amadan

carlos / platform Public

instance-control: rootfs start, stop, cmd and sweep from a root-resolved manifest command (rootfs apps plan 2, task 5)

Root starts a rootfs unit only from the host's own staged image, reached
beneath the version store with no symlink and hashed against the manifest
root staged beside it (C3), and resolves the command from that manifest,
never from anything the instance can write (C4). The exe layout is
<ExeRoot>/<key>/rootfs/<role>/{image, launch.json, resolv.conf}; gen and
worker units get their drop-in and one view read, with a daemon-reload
only when the view does not apply, the jit drop-in changed or a reload is
owed (R11); after every start the unit's control group and its
MemoryDenyWriteExecute are checked and a mismatch is stopped (C5, C10).

role=worker is a rootfs role only: every verb refuses it without
runtime=rootfs. The sweep lists the rootfs templates and judges a rootfs
generation's exe directory by the rootfs gen unit. The claim drop-in read
no longer admits rootfs units; the activator's cgroup wait refuses an
empty unit name. Client verbs: StartRootfs, StartRootfsWorker, StopRootfs,
CmdRootfs, CmdStatusRootfs, ReadyRootfs. The #328 egress slot is marked
in both start functions.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 0a59e97be82c8d197100818fcc81743819dbcd51 parent 22a4e30
19 files changed, +3496 −85
  • approved_rootfs_copy_test.go +2 −0
  • internal/activator/backend_exec.go +5 −0
  • internal/activator/rootfs_worker_unit_test.go +23 −0
  • internal/host/claimstop_run.go +20 −1
  • internal/host/claimunit_dropin.go +17 −0
  • internal/host/claimunit_dropin_linux.go +1 −3
  • internal/host/claimunit_dropin_test.go +31 −0
  • internal/instancectl/claim.go +3 −1
  • internal/instancectl/client.go +132 −4
  • internal/instancectl/cmd.go +30 −6
  • internal/instancectl/gen.go +132 −30
  • internal/instancectl/names_gen_test.go +2 −1
  • internal/instancectl/rootfs.go +1046 −0
  • internal/instancectl/rootfs_identity_linux.go +22 −0
  • internal/instancectl/rootfs_identity_other.go +12 −0
  • internal/instancectl/rootfs_test.go +1751 −0
  • internal/instancectl/server.go +214 −31
  • internal/instancectl/server_gen_test.go +3 −1
  • internal/instancectl/server_test.go +50 −7
diff --git a/approved_rootfs_copy_test.go b/approved_rootfs_copy_test.go
index 22509e345..9cab0b707 100644
--- a/approved_rootfs_copy_test.go
+++ b/approved_rootfs_copy_test.go
@@ -173,6 +173,8 @@ var rootfsCopyBuilt = []string{
"RFS-1-C90", "RFS-1-C91", "RFS-1-C92", "RFS-1-C93", "RFS-1-C94", "RFS-1-C95", "RFS-1-C96", "RFS-1-C97", "RFS-1-C98",
// Final review: the scratch pool's per-account refusal names no app (internal/console/seedscratch.go)
"RFS-1-C99",
+ // Plan 2 Task 5: the control service's rootfs start (internal/instancectl/rootfs.go)
+ "RFS-2-C07", "RFS-2-C08", "RFS-2-C10",
}
func mustParseRootfsRecord(t *testing.T) []copyRow {
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go
index d6c81c739..b830c5313 100644
--- a/internal/activator/backend_exec.go
+++ b/internal/activator/backend_exec.go
@@ -1654,6 +1654,11 @@ func (b *ExecBackend) waitRoleCgroupEmpty(ctx context.Context, rt registry.Route
}
slice := instancectl.SliceName(rt.Host)
unit := instancectl.UnitName(role, rt.Host)
+ if unit == "" {
+ // A worker has no binary unit (rootfs apps plan 2): "" would name
+ // the slice's own cgroup.events and read every unit as this one.
+ return fmt.Errorf("%s: role %s has no binary unit", rt.Host, role)
+ }
bound := roleEmptyBound
deadline := time.Now().Add(bound)
for {
diff --git a/internal/activator/rootfs_worker_unit_test.go b/internal/activator/rootfs_worker_unit_test.go
new file mode 100644
index 000000000..42242a944
--- /dev/null
+++ b/internal/activator/rootfs_worker_unit_test.go
@@ -0,0 +1,23 @@
+package activator
+
+import (
+ "context"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/instancectl"
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// A worker has no binary unit (rootfs apps plan 2, Task 1): UnitName
+// answers "", and a wait keyed by "" would read the slice's own
+// cgroup.events as the unit's. The wait refuses instead of reading.
+func TestWaitRoleCgroupEmptyRefusesAWorker(t *testing.T) {
+ t.Parallel()
+ b := &ExecBackend{CgroupEvents: func(slice, unit string) (bool, bool, error) {
+ t.Errorf("cgroup.events read for slice %q unit %q", slice, unit)
+ return false, false, nil
+ }}
+ if err := b.waitRoleCgroupEmpty(context.Background(), registry.Route{Host: "shop.example.com"}, instancectl.RoleWorker); err == nil {
+ t.Fatal("a worker's wait was not refused")
+ }
+}
diff --git a/internal/host/claimstop_run.go b/internal/host/claimstop_run.go
index ed8b38c33..a32c561c7 100644
--- a/internal/host/claimstop_run.go
+++ b/internal/host/claimstop_run.go
@@ -33,7 +33,7 @@ func RunInstanceControl(args []string) {
log.Fatal(err)
}
if !hostwrite.Enabled(claimEntryEnv(ctx)) {
- instancectl.RunWithRegistry(args, instanceRegistryHostApp(nil))
+ instancectl.RunWithRegistryView(args, instancectl.RegistryView{HostApp: instanceRegistryHostApp(nil), HostRuntime: instanceRegistryHostRuntime})
return
}
reg, err := openServiceRegistry(filepath.Join(dataDir(), "registry.db"))
@@ -140,6 +140,25 @@ func instanceRegistryHostApp(shared *registry.Registry) func(string) (string, st
}
}
+// instanceRegistryHostRuntime reads a host's Route.Runtime from the
+// registry for the control service's rootfs starts (rootfs apps plan 2):
+// only a rootfs row is ever started from an image.
+func instanceRegistryHostRuntime(host string) (string, error) {
+ reg, err := openServiceRegistryReadOnly(filepath.Join(dataDir(), "registry.db"))
+ if err != nil {
+ return "", err
+ }
+ defer reg.Close()
+ row, err := reg.Get(host)
+ if err != nil {
+ return "", err
+ }
+ if row == nil {
+ return "", os.ErrNotExist
+ }
+ return row.Runtime, nil
+}
+
func claimActivationReady(ctx context.Context, open func() (rootparticipation.Ledger, bool, error)) error {
l, required, err := open()
if l.State != nil {
diff --git a/internal/host/claimunit_dropin.go b/internal/host/claimunit_dropin.go
new file mode 100644
index 000000000..4858adf1a
--- /dev/null
+++ b/internal/host/claimunit_dropin.go
@@ -0,0 +1,17 @@
+package host
+
+import "github.com/carlosframework/platform/internal/instancectl"
+
+// claimDropinUnit reports whether unit is one whose claim launch drop-in
+// (60-claim-launch.conf) the effective-shape check may read: a bound claim
+// unit, or the host's ordinary binary run unit (a generation's included).
+// A rootfs unit never is: the claim launch knows no image, so no rootfs
+// unit is ever a claim unit, and ParseUnitName's admitting them as RoleRun
+// (rootfs apps plan 2, Task 1) must not widen this read.
+func claimDropinUnit(unit string) bool {
+ if instancectl.ValidBoundClaimUnit(unit) {
+ return true
+ }
+ role, _, ordinary := instancectl.ParseUnitName(unit)
+ return ordinary && role == instancectl.RoleRun && !instancectl.IsRootfsUnit(unit)
+}
diff --git a/internal/host/claimunit_dropin_linux.go b/internal/host/claimunit_dropin_linux.go
index 4bbbdc7ca..c5d5afb20 100644
--- a/internal/host/claimunit_dropin_linux.go
+++ b/internal/host/claimunit_dropin_linux.go
@@ -1,7 +1,6 @@
package host
import (
- "github.com/carlosframework/platform/internal/instancectl"
"github.com/carlosframework/platform/internal/rootstate"
"github.com/carlosframework/platform/internal/safefile"
"golang.org/x/sys/unix"
@@ -9,8 +8,7 @@ import (
)
func openClaimUnitDropin(unit string) (*os.File, error) {
- role, _, ordinary := instancectl.ParseUnitName(unit)
- if !instancectl.ValidBoundClaimUnit(unit) && !(ordinary && role == instancectl.RoleRun) {
+ if !claimDropinUnit(unit) {
return nil, os.ErrPermission
}
const base = "/run/systemd/system"
diff --git a/internal/host/claimunit_dropin_test.go b/internal/host/claimunit_dropin_test.go
new file mode 100644
index 000000000..00fd2ed6b
--- /dev/null
+++ b/internal/host/claimunit_dropin_test.go
@@ -0,0 +1,31 @@
+package host
+
+import (
+ "testing"
+
+ "github.com/carlosframework/platform/internal/instancectl"
+)
+
+// The claim launch drop-in is read for a bound claim unit or the host's
+// binary run unit only. A rootfs run or generation unit parses as RoleRun
+// since rootfs apps plan 2 Task 1, and must stay out: the claim launch
+// knows no image.
+func TestClaimDropinUnitRefusesRootfsUnits(t *testing.T) {
+ t.Parallel()
+ host := "shop.example.com"
+ for _, unit := range []string{instancectl.UnitName(instancectl.RoleRun, host), instancectl.GenUnitName(host, 3), "console.service"} {
+ if !claimDropinUnit(unit) {
+ t.Errorf("%s refused", unit)
+ }
+ }
+ for _, unit := range []string{
+ instancectl.RootfsUnitName(instancectl.RoleRun, host), instancectl.RootfsGenUnitName(host, 3),
+ instancectl.WorkerUnitName(host, "jobs"), instancectl.RootfsUnitName(instancectl.RoleCmd, host),
+ instancectl.UnitName(instancectl.RoleSeed, host), instancectl.UnitName(instancectl.RoleCmd, host),
+ "carlos-instance@.service", "",
+ } {
+ if claimDropinUnit(unit) {
+ t.Errorf("%q accepted", unit)
+ }
+ }
+}
diff --git a/internal/instancectl/claim.go b/internal/instancectl/claim.go
index bd829c6b9..febb96a18 100644
--- a/internal/instancectl/claim.go
+++ b/internal/instancectl/claim.go
@@ -20,5 +20,7 @@ func requestConnContext(ctx context.Context, c net.Conn) context.Context {
// heldRelease is not nil, the moves held-release route through it (the
// host passes it only with both flags on; nil leaves the route absent).
func RunWithClaim(args []string, handler, heldRelease http.Handler, hostApp func(string) (string, string, error)) {
- runWithClaim(args, handler, heldRelease, hostApp)
+ // The claim root reads account and app only: under claim control no
+ // rootfs unit starts (RFS-2-C06), so the service needs no runtime view.
+ runWithClaim(args, handler, heldRelease, RegistryView{HostApp: hostApp})
}
diff --git a/internal/instancectl/client.go b/internal/instancectl/client.go
index 5b233fbf0..a3d653c1b 100644
--- a/internal/instancectl/client.go
+++ b/internal/instancectl/client.go
@@ -1,6 +1,7 @@
package instancectl
import (
+ "bytes"
"context"
"encoding/json"
"errors"
@@ -98,23 +99,35 @@ func (c *Client) httpClient() *http.Client {
// message; a dial failure names the socket so a forgotten enable reads as
// what it is from carlos logs.
func (c *Client) do(ctx context.Context, deadline time.Duration, method, path string, q url.Values, out any) (int, http.Header, error) {
+ return c.doBody(ctx, deadline, method, path, q, nil, out)
+}
+
+// doBody is do with a request body (a rootfs cmd's argv).
+func (c *Client) doBody(ctx context.Context, deadline time.Duration, method, path string, q url.Values, payload []byte, out any) (int, http.Header, error) {
ctx, cancel := context.WithTimeout(ctx, deadline)
defer cancel()
u := "http://control" + path
if len(q) > 0 {
u += "?" + q.Encode()
}
- req, err := http.NewRequestWithContext(ctx, method, u, nil)
+ var rd io.Reader
+ if payload != nil {
+ rd = bytes.NewReader(payload)
+ }
+ req, err := http.NewRequestWithContext(ctx, method, u, rd)
if err != nil {
return 0, nil, err
}
+ if payload != nil {
+ req.Header.Set("Content-Type", "application/json")
+ }
resp, err := c.httpClient().Do(req)
if err != nil {
return 0, nil, fmt.Errorf("instance control socket %s: %w", c.Path, err)
}
defer resp.Body.Close()
body, _ := io.ReadAll(io.LimitReader(resp.Body, 64<<10))
- if resp.StatusCode == http.StatusServiceUnavailable && path == "/ready" {
+ if resp.StatusCode == http.StatusServiceUnavailable && (path == "/ready" || path == RootfsReadyPath) {
var r ProbeReport
_ = json.Unmarshal(body, &r)
return resp.StatusCode, resp.Header, &NotReadyError{Report: r}
@@ -336,8 +349,15 @@ func (c *Client) Sweep(ctx context.Context) (SweepResponse, error) {
// *NotReadyError on 503, and a *StaleBuildError on a 204 whose BuildHeader
// is missing or names another build (#256). The verdict is asked first:
// a service that is both stale and not ready says why it is not ready.
-func (c *Client) Ready(ctx context.Context) error {
- _, hdr, err := c.do(ctx, DeadlineReady, http.MethodGet, "/ready", nil, nil)
+func (c *Client) Ready(ctx context.Context) error { return c.ready(ctx, "/ready") }
+
+// ReadyRootfs is Ready for the rootfs verdict (GET /ready/rootfs): both
+// probes passed on this boot, or a *NotReadyError carrying the rootfs
+// report (rootfs apps plan 2).
+func (c *Client) ReadyRootfs(ctx context.Context) error { return c.ready(ctx, RootfsReadyPath) }
+
+func (c *Client) ready(ctx context.Context, path string) error {
+ _, hdr, err := c.do(ctx, DeadlineReady, http.MethodGet, path, nil, nil)
if err != nil {
return err
}
@@ -346,3 +366,111 @@ func (c *Client) Ready(ctx context.Context) error {
}
return nil
}
+
+// The rootfs verbs (rootfs apps plan 2). Every one carries runtime=rootfs;
+// the launch verbs name the staged image and root resolves the command from
+// its own staged manifest. A 403 is root's refusal before anything was laid
+// out, reported as ErrStagedImage (preflight 1.8), so the agent can tell it
+// from a start that failed.
+
+// setStagedImage puts img on a rootfs launch query.
+func setStagedImage(q url.Values, img StagedImage) {
+ q.Set("runtime", runtimeRootfs)
+ q.Set("image", img.Path)
+ q.Set("version", img.Version)
+ q.Set("digest", img.Digest)
+ if img.Process != "" {
+ q.Set("process", img.Process)
+ }
+}
+
+// rootfsStopValues is the query of a rootfs termination verb: the role, the
+// generation of a run role, the name of a worker, and runtime=rootfs.
+func rootfsStopValues(host string, role Role, gen uint64, worker string) url.Values {
+ q := hostRole(host, role)
+ q.Set("runtime", runtimeRootfs)
+ if gen != 0 {
+ q.Set("gen", strconv.FormatUint(gen, 10))
+ }
+ if worker != "" {
+ q.Set("worker", worker)
+ }
+ return q
+}
+
+// StartRootfs starts the rootfs run unit of host (gen 0) or its generation
+// gen from the staged image, whose Process is web.
+func (c *Client) StartRootfs(ctx context.Context, host string, gen uint64, memoryMax int64, img StagedImage) (string, error) {
+ q := hostRole(host, RoleRun)
+ if gen != 0 {
+ q.Set("gen", strconv.FormatUint(gen, 10))
+ }
+ return c.startRootfsQuery(ctx, q, memoryMax, img)
+}
+
+// StartRootfsWorker starts the named worker of host from the staged image,
+// whose Process is the worker's name.
+func (c *Client) StartRootfsWorker(ctx context.Context, host, name string, memoryMax int64, img StagedImage) (string, error) {
+ q := hostRole(host, RoleWorker)
+ q.Set("worker", name)
+ return c.startRootfsQuery(ctx, q, memoryMax, img)
+}
+
+func (c *Client) startRootfsQuery(ctx context.Context, q url.Values, memoryMax int64, img StagedImage) (string, error) {
+ q.Set("memory_max", strconv.FormatInt(memoryMax, 10))
+ setStagedImage(q, img)
+ var out StartResponse
+ status, _, err := c.do(ctx, DeadlineControl, http.MethodPost, "/start", q, &out)
+ if err != nil {
+ switch {
+ case status == http.StatusForbidden:
+ return "", fmt.Errorf("%w: %v", ErrStagedImage, err)
+ case status == statusGenUnproven && q.Has("gen"):
+ return "", fmt.Errorf("%w: %w", ErrGenUnproven, err)
+ }
+ return "", err
+ }
+ return out.InvocationID, nil
+}
+
+// StopRootfs stops one rootfs unit of host: the run unit (gen 0), a
+// generation, the named worker, or the cmd unit.
+func (c *Client) StopRootfs(ctx context.Context, host string, role Role, gen uint64, worker string) error {
+ _, _, err := c.do(ctx, DeadlineControl, http.MethodPost, "/stop", rootfsStopValues(host, role, gen, worker), nil)
+ return err
+}
+
+// CmdRootfs starts host's rootfs cmd unit (run spec §5.4 for a rootfs
+// row): img.Process names a declared process (release, or a worker for a
+// one-off run), or img.Argv is an operator's command. It returns the new
+// invocation.
+func (c *Client) CmdRootfs(ctx context.Context, host string, memoryMax int64, img StagedImage) (string, error) {
+ q := hostRole(host, "")
+ q.Set("memory_max", strconv.FormatInt(memoryMax, 10))
+ setStagedImage(q, img)
+ var body []byte
+ if len(img.Argv) > 0 {
+ var err error
+ if body, err = json.Marshal(struct {
+ Argv []string `json:"argv"`
+ }{img.Argv}); err != nil {
+ return "", err
+ }
+ }
+ var out StartResponse
+ status, _, err := c.doBody(ctx, DeadlineControl, http.MethodPost, "/cmd", q, body, &out)
+ if err != nil {
+ if status == http.StatusForbidden {
+ return "", fmt.Errorf("%w: %v", ErrStagedImage, err)
+ }
+ return "", err
+ }
+ return out.InvocationID, nil
+}
+
+// CmdStatusRootfs is CmdStatus for the rootfs cmd unit.
+func (c *Client) CmdStatusRootfs(ctx context.Context, host, invocation string) (CmdStatus, error) {
+ var out CmdStatus
+ _, _, err := c.do(ctx, DeadlineCmdPoll, http.MethodGet, "/cmd/"+url.PathEscape(host), url.Values{"invocation": {invocation}, "runtime": {runtimeRootfs}}, &out)
+ return out, err
+}
diff --git a/internal/instancectl/cmd.go b/internal/instancectl/cmd.go
index ffd7de516..ca461edbc 100644
--- a/internal/instancectl/cmd.go
+++ b/internal/instancectl/cmd.go
@@ -39,11 +39,15 @@ func (s *Server) cmdStartWait() time.Duration {
// is refused, never restarted. A failed one is reset first, or start would
// keep its old result.
func (s *Server) handleCmd(w http.ResponseWriter, r *http.Request) {
- req, err := parseRequest(r, verbShape{tier: true, exe: true, cmd: true})
+ req, err := parseRequest(r, verbShape{tier: true, exe: true, cmd: true, rootfs: true})
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
+ if req.rootfs() {
+ s.startRootfsCmd(w, r, req)
+ return
+ }
// One start per host at a time: a second request waits here, then reads
// the unit loaded and is refused, instead of racing the first one's
// invocation check and stopping its command.
@@ -125,8 +129,19 @@ func (s *Server) handleCmdStatus(w http.ResponseWriter, r *http.Request) {
http.Error(w, "invocation is required", http.StatusBadRequest)
return
}
+ // The poll follows the termination verbs' rule (Astra R1 finding 6):
+ // runtime=rootfs alone picks the rootfs cmd unit, an image field is a
+ // 400, and role=worker is refused as everywhere.
+ req, err := parseQuery(host, r.URL.Query(), verbShape{rootfsStop: true})
+ if err != nil {
+ http.Error(w, err.Error(), http.StatusBadRequest)
+ return
+ }
ctx := r.Context()
unit := UnitName(RoleCmd, host)
+ if req.rootfs() {
+ unit = RootfsUnitName(RoleCmd, host)
+ }
deadline := time.Now().Add(s.cmdWait())
for {
out, err := s.Systemctl(ctx, "show", "-p", "InvocationID,ActiveState,SubState,Result,ExecMainStatus,ExecMainCode", unit)
@@ -163,15 +178,24 @@ func cmdStatusOf(m map[string]string, want string) CmdStatus {
}
// cleanCmdExe removes host's cmd executable sources after its cmd unit has
-// stopped. It holds the host's cmd lock, so a start cannot be laying them
-// out meanwhile, and leaves them for a unit that is not inactive or failed
-// (a start that has since begun).
-func (s *Server) cleanCmdExe(ctx context.Context, host string) {
+// stopped: the binary cmd unit's two links, or with rootfs the rootfs cmd
+// unit's layout (<iid>/rootfs/cmd) and its jit drop-in. It holds the host's
+// cmd lock, so a start cannot be laying them out meanwhile, and leaves them
+// for a unit that is not inactive or failed (a start that has since begun).
+func (s *Server) cleanCmdExe(ctx context.Context, host string, rootfs bool) {
lock, _ := s.cmdLocks.LoadOrStore(host, new(sync.Mutex))
lock.(*sync.Mutex).Lock()
defer lock.(*sync.Mutex).Unlock()
- switch s.showValue(ctx, "ActiveState", UnitName(RoleCmd, host)) {
+ unit := UnitName(RoleCmd, host)
+ if rootfs {
+ unit = RootfsUnitName(RoleCmd, host)
+ }
+ switch s.showValue(ctx, "ActiveState", unit) {
case "inactive", "failed":
+ if rootfs {
+ s.removeRootfsFiles(unit)
+ return
+ }
s.removeCmdExe(provision.InstanceID(host))
}
}
diff --git a/internal/instancectl/gen.go b/internal/instancectl/gen.go
index 154da8771..f2ea69c56 100644
--- a/internal/instancectl/gen.go
+++ b/internal/instancectl/gen.go
@@ -39,18 +39,39 @@ const DropInName = "50-carlos-generation.conf"
// The memory lines are the generation's own tier (spec §7.3). The
// template's OOMPolicy=kill gives the unit's cgroup memory.oom.group=1.
func GenDropIn(host string, n uint64, tier cgroup.Limit) string {
- iid := provision.InstanceID(host)
key := GenKey(host, n)
- max := strconv.FormatInt(tier.MemoryMax, 10)
+ return genDropInHeader(host) +
+ "BindReadOnlyPaths=" + exeRoot + key + "/app:/opt/carlos/app " + exeRoot + key + "/live:/opt/carlos/live\n" +
+ "StandardOutput=file:" + GenLogSocket(host, n) + "\n" +
+ tierLines(tier)
+}
+
+// exeRoot is the control service's exe root as the templates and drop-ins
+// name it, literally.
+const exeRoot = "/run/carlos-control/exe/"
+
+// genDropInHeader is the part of a generation drop-in that follows from the
+// host alone: its slice and the instance's own two directories. The binary
+// and rootfs generation drop-ins share it, so the two cannot drift.
+func genDropInHeader(host string) string {
return "# Written by carlos instance-control for one generation; removed when it stops (zero-downtime plan 2).\n" +
- "[Service]\n" +
- "Slice=" + SliceName(host) + "\n" +
+ "[Service]\n" + hostMountLines(host)
+}
+
+// hostMountLines are the Slice=, BindPaths=, ReadWritePaths= and
+// WorkingDirectory= lines every per-start drop-in carries for host.
+func hostMountLines(host string) string {
+ iid := provision.InstanceID(host)
+ return "Slice=" + SliceName(host) + "\n" +
"BindPaths=/data/instances/" + iid + " /run/carlos/instances/" + iid + "\n" +
"ReadWritePaths=/data/instances/" + iid + " /run/carlos/instances/" + iid + "\n" +
- "WorkingDirectory=/data/instances/" + iid + "\n" +
- "BindReadOnlyPaths=/run/carlos-control/exe/" + key + "/app:/opt/carlos/app /run/carlos-control/exe/" + key + "/live:/opt/carlos/live\n" +
- "StandardOutput=file:" + GenLogSocket(host, n) + "\n" +
- "MemoryMax=" + max + "\n" +
+ "WorkingDirectory=/data/instances/" + iid + "\n"
+}
+
+// tierLines are a generation's own memory lines (spec §7.3).
+func tierLines(tier cgroup.Limit) string {
+ max := strconv.FormatInt(tier.MemoryMax, 10)
+ return "MemoryMax=" + max + "\n" +
"MemoryHigh=" + strconv.FormatInt(cgroup.HighOf(tier.MemoryMax), 10) + "\n" +
"MemorySwapMax=" + max + "\n"
}
@@ -58,15 +79,32 @@ func GenDropIn(host string, n uint64, tier cgroup.Limit) string {
// writeGenDropIn writes the drop-in atomically: a temp file systemd never
// reads (only *.conf is read), renamed into place.
func (s *Server) writeGenDropIn(host string, n uint64, tier cgroup.Limit) error {
- dir := filepath.Join(s.DropInDir, GenUnitName(host, n)+".d")
+ return s.writeDropIn(GenUnitName(host, n), DropInName, GenDropIn(host, n, tier))
+}
+
+// writeDropIn writes <DropInDir>/<unit>.d/<name> atomically: a temp file
+// systemd never reads (only *.conf is read), renamed into place.
+func (s *Server) writeDropIn(unit, name, body string) error {
+ dir := filepath.Join(s.DropInDir, unit+".d")
if err := os.MkdirAll(dir, 0o755); err != nil {
return err
}
- tmp := filepath.Join(dir, "."+DropInName+".tmp")
- if err := os.WriteFile(tmp, []byte(GenDropIn(host, n, tier)), 0o644); err != nil {
+ tmp := filepath.Join(dir, "."+name+".tmp")
+ if err := os.WriteFile(tmp, []byte(body), 0o644); err != nil {
return err
}
- return os.Rename(tmp, filepath.Join(dir, DropInName))
+ return os.Rename(tmp, filepath.Join(dir, name))
+}
+
+// writeDropInIfChanged is writeDropIn unless the file already holds body:
+// rewriting identical bytes moves the mtime, and systemd then reports
+// NeedDaemonReload=yes for a loaded unit whose name is reused (a worker's).
+// It reports whether it wrote.
+func (s *Server) writeDropInIfChanged(unit, name, body string) (bool, error) {
+ if cur, err := os.ReadFile(filepath.Join(s.DropInDir, unit+".d", name)); err == nil && string(cur) == body {
+ return false, nil
+ }
+ return true, s.writeDropIn(unit, name, body)
}
// ensureGenExe lays out generation n's view of its build under ExeRoot,
@@ -78,14 +116,21 @@ func (s *Server) ensureGenExe(host string, n uint64) error {
}
// removeGenFiles removes a generation unit's drop-in and executable layout,
-// by the unit's name as systemd reports it. Best-effort: a leftover is a
-// few bytes in /run, never a running unit, so most callers drop the error.
-// The sweep logs it, so a leftover it cannot remove says why (Task 15).
+// by the unit's name as systemd reports it, for a binary or a rootfs
+// generation. Best-effort: a leftover is a few bytes in /run, never a
+// running unit, so most callers drop the error. The sweep logs it, so a
+// leftover it cannot remove says why (Task 15).
func (s *Server) removeGenFiles(unit string) error {
esc, n, ok := ParseGenUnitName(unit)
+ if !ok {
+ esc, n, ok = ParseRootfsGenUnitName(unit)
+ }
if !ok {
return nil
}
+ // The .d directory holds the jit drop-in too (R11): its removal is
+ // counted, so the next rootfs start reloads.
+ s.removeJITDropIn(unit)
var errs []error
if s.DropInDir != "" {
errs = append(errs, os.RemoveAll(filepath.Join(s.DropInDir, unit+".d")))
@@ -136,15 +181,22 @@ func (s *Server) genView(ctx context.Context, host, unit string, tier cgroup.Lim
applied = m["Slice"] == SliceName(host) && m["NeedDaemonReload"] == "no" &&
m["MemoryMax"] == max && m["MemorySwapMax"] == max && m["MemoryHigh"] == high
if len(blocks) == 2 {
- sl := parseShow(blocks[1])
- sh := cgroup.SharesOf(tier)
- tierSet = sl["MemoryMax"] == max && sl["MemorySwapMax"] == max && sl["MemoryHigh"] == high &&
- sl["CPUWeight"] == strconv.Itoa(sh.CPUWeight) && sl["IOWeight"] == strconv.Itoa(sh.IOWeight) &&
- sl["CPUQuotaPerSecUSec"] == cpuQuotaShown(sh.CPUQuotaPercent)
+ tierSet = sliceAtTier(parseShow(blocks[1]), tier)
}
return m["Slice"], applied, tierSet
}
+// sliceAtTier reports whether a slice's shown properties already carry
+// every value setTier would give it.
+func sliceAtTier(sl map[string]string, tier cgroup.Limit) bool {
+ max := strconv.FormatInt(tier.MemoryMax, 10)
+ high := strconv.FormatInt(cgroup.HighOf(tier.MemoryMax), 10)
+ sh := cgroup.SharesOf(tier)
+ return sl["MemoryMax"] == max && sl["MemorySwapMax"] == max && sl["MemoryHigh"] == high &&
+ sl["CPUWeight"] == strconv.Itoa(sh.CPUWeight) && sl["IOWeight"] == strconv.Itoa(sh.IOWeight) &&
+ sl["CPUQuotaPerSecUSec"] == cpuQuotaShown(sh.CPUQuotaPercent)
+}
+
// cpuQuotaShown is CPUQuota=<pct>% as systemctl show prints its
// CPUQuotaPerSecUSec (systemd 257): 500ms, 1s, 1.500000s. A format this
// gets wrong only costs the set-property call it would have saved.
@@ -252,6 +304,12 @@ func (s *Server) startGen(w http.ResponseWriter, r *http.Request, req request) {
// unit's cgroup empty removes its files; otherwise they stay, and the
// answer is statusGenUnproven so the client knows the generation may live.
func (s *Server) abandonGen(ctx context.Context, w http.ResponseWriter, host, unit, msg string) {
+ s.abandonUnit(ctx, w, host, unit, msg, func() { s.removeGenFiles(unit) })
+}
+
+// abandonUnit is abandonGen for any per-start unit: remove is what takes
+// its files once the unit is proven gone.
+func (s *Server) abandonUnit(ctx context.Context, w http.ResponseWriter, host, unit, msg string, remove func()) {
dctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), DeadlineControl+s.waitBudget())
defer cancel()
if err := s.killGen(dctx, host, unit); err != nil {
@@ -260,7 +318,7 @@ func (s *Server) abandonGen(ctx context.Context, w http.ResponseWriter, host, un
return
}
s.resetGenFailed(dctx, unit)
- s.removeGenFiles(unit)
+ remove()
http.Error(w, msg, http.StatusBadGateway)
}
@@ -346,6 +404,11 @@ func (s *Server) markGenStarting(unit string, on bool) {
// own directory (<iid>, no '@') is never a generation key and is never
// touched. On a box that never ran generations, ExeRoot holds only host
// directories and this finds nothing.
+//
+// Rootfs generations and workers (rootfs apps plan 2) are swept the same
+// way: a rootfs generation's exe directory is judged by the rootfs gen unit
+// (genUnitOfExeKey, preflight 1.2), so one whose stop failed keeps its
+// files, and a worker's directory (<iid>@<name>) by its worker unit.
func (s *Server) sweepGenLeftovers(ctx context.Context, failed map[string]string) {
var units []string
seen := map[string]bool{}
@@ -361,16 +424,27 @@ func (s *Server) sweepGenLeftovers(ctx context.Context, failed map[string]string
if !ok || !e.IsDir() {
continue
}
- if _, _, ok := ParseGenUnitName(unit); !ok {
+ if _, _, ok := ParseGenUnitName(unit); ok {
+ add(unit)
+ continue
+ }
+ if _, _, ok := ParseRootfsGenUnitName(unit); ok {
+ add(unit)
continue
}
- add(unit)
+ if _, _, ok := ParseWorkerUnitName(unit); ok {
+ add(unit)
+ }
}
for _, e := range s.readSweepDir(s.ExeRoot) {
if !e.IsDir() {
continue
}
- if unit, ok := genUnitOfExeKey(e.Name()); ok {
+ if unit, ok := s.genUnitOfExeKey(e.Name()); ok {
+ add(unit)
+ continue
+ }
+ if unit, ok := workerUnitOfExeKey(e.Name()); ok {
add(unit)
}
}
@@ -393,15 +467,21 @@ func (s *Server) readSweepDir(dir string) []os.DirEntry {
}
// genUnitOfExeKey names the generation unit whose executable directory is
-// key. It answers only when removeGenFiles, given that unit, removes
-// exactly key, so the sweep can never remove some other directory.
-func genUnitOfExeKey(key string) (string, bool) {
+// key: the rootfs generation unit when the directory holds a rootfs layout
+// (<key>/rootfs/), the binary one otherwise (preflight 1.2), so the sweep
+// judges the directory by the unit that ran from it. It answers only when
+// removeGenFiles, given that unit, removes exactly key, so the sweep can
+// never remove some other directory.
+func (s *Server) genUnitOfExeKey(key string) (string, bool) {
iid, n, ok := ParseGenKey(key)
if !ok {
return "", false
}
- unit := GenUnitNameOf(iid, n)
- esc, m, ok := ParseGenUnitName(unit)
+ unit, parse := GenUnitNameOf(iid, n), ParseGenUnitName
+ if fi, err := os.Lstat(filepath.Join(s.ExeRoot, key, RootfsExeDirName)); err == nil && fi.IsDir() {
+ unit, parse = RootfsGenUnitNameOf(iid, n), ParseRootfsGenUnitName
+ }
+ esc, m, ok := parse(unit)
if !ok || m != n {
return "", false
}
@@ -411,6 +491,24 @@ func genUnitOfExeKey(key string) (string, bool) {
return unit, true
}
+// workerUnitOfExeKey is genUnitOfExeKey for a worker's directory
+// (<iid>@<name>): the worker unit whose files removeWorkerFiles removes.
+func workerUnitOfExeKey(key string) (string, bool) {
+ iid, name, ok := ParseWorkerKey(key)
+ if !ok {
+ return "", false
+ }
+ unit := WorkerUnitNameOf(iid, name)
+ esc, back, ok := ParseWorkerUnitName(unit)
+ if !ok || back != name {
+ return "", false
+ }
+ if raw, ok := Unescape(esc); !ok || WorkerKeyOf(raw, name) != key {
+ return "", false
+ }
+ return unit, true
+}
+
func (s *Server) sweepGenLeftover(ctx context.Context, unit string) {
// Held across the check and the removal, so a start of the same unit
// cannot write fresh files in between.
@@ -429,7 +527,11 @@ func (s *Server) sweepGenLeftover(ctx context.Context, unit string) {
if err := WaitCgroupEmpty(wctx, s.CgroupRoot, "carlos-instances-"+esc+".slice", unit, time.Millisecond); err != nil {
return
}
- if err := s.removeGenFiles(unit); err != nil {
+ remove := s.removeGenFiles
+ if _, _, ok := ParseWorkerUnitName(unit); ok {
+ remove = s.removeWorkerFiles
+ }
+ if err := remove(unit); err != nil {
s.logf("instance-control: sweep: could not remove the files of %s: %v", unit, err)
return
}
diff --git a/internal/instancectl/names_gen_test.go b/internal/instancectl/names_gen_test.go
index 3dd270c76..d9dd732db 100644
--- a/internal/instancectl/names_gen_test.go
+++ b/internal/instancectl/names_gen_test.go
@@ -134,7 +134,8 @@ func TestParseGenKeyInvertsGenKey(t *testing.T) {
if got := GenUnitNameOf(iid, n); got != GenUnitName(host, 7) {
t.Errorf("GenUnitNameOf = %q, want %q", got, GenUnitName(host, 7))
}
- if unit, ok := genUnitOfExeKey(GenKey(host, 7)); !ok || unit != GenUnitName(host, 7) {
+ // No rootfs layout under the key: the binary generation's unit.
+ if unit, ok := (&Server{ExeRoot: t.TempDir()}).genUnitOfExeKey(GenKey(host, 7)); !ok || unit != GenUnitName(host, 7) {
t.Errorf("genUnitOfExeKey(GenKey(%q, 7)) = %q, %v", host, unit, ok)
}
}
diff --git a/internal/instancectl/rootfs.go b/internal/instancectl/rootfs.go
new file mode 100644
index 000000000..8c0e80387
--- /dev/null
+++ b/internal/instancectl/rootfs.go
@@ -0,0 +1,1046 @@
+package instancectl
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "maps"
+ "net/http"
+ "os"
+ "path"
+ "path/filepath"
+ "slices"
+ "strconv"
+ "strings"
+ "sync"
+ "time"
+
+ "github.com/carlosframework/platform/internal/cgroup"
+ "github.com/carlosframework/platform/internal/instancefs"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// The rootfs start path (rootfs apps design §5, plan 2 Task 5). A rootfs
+// instance runs from a squashfs image root mounts into its unit's
+// namespace, and the command it runs comes from root's own staged manifest
+// (C4): the agent names the staged image, root checks that it is the
+// host's own release (C3), resolves the process from the manifest it
+// staged beside it, writes the launch under its exe root, and only then
+// starts the unit, checking afterwards that systemd placed it in the
+// host's slice (C5) and gave it the MemoryDenyWriteExecute the release's
+// jit asks for (C10).
+
+// RegistryView is what the control service reads of the registry: a
+// host's account and app, and its runtime.
+type RegistryView struct {
+ HostApp func(host string) (acct, app string, err error)
+ HostRuntime func(host string) (string, error)
+}
+
+// boxPlatform is release.BoxPlatform, a var so a test on another platform
+// can name the key rootfs manifests carry.
+var boxPlatform = release.BoxPlatform
+
+// The approved copy this file speaks (docs/superpowers/plans/2026-10-24-rootfs-apps-copy.md).
+const (
+ // RFS-2-C02; the deck's cell is internal/host/rootfsplacement.go, which
+ // carries the same text.
+ rootfsProbeNotPassedText = "this box has not passed the rootfs isolation probe on this boot (%s)"
+ // RFS-2-C07
+ rootfsNotRootfsText = "the staged release for this instance is a binary, not a rootfs image"
+ // RFS-2-C08
+ rootfsUndeclaredText = "%s is not a process this release declares"
+ // RFS-2-C10
+ rootfsMissingPathText = "the image has no %s, which the instance needs; rebuild it with carlos ship"
+)
+
+// The two shows of a rootfs start, as systemctl is called.
+const (
+ // rootfsPostStartShow is read after every rootfs start: the invocation,
+ // where systemd put the unit, and the one property jit changes.
+ rootfsPostStartShow = "show -p InvocationID,Slice,ControlGroup,MemoryDenyWriteExecute"
+ // rootfsViewShow is genView's shape plus RootImage, for a gen or worker
+ // unit and its host slice (preflight 2.11).
+ rootfsViewShow = "show -p Slice,RootImage,NeedDaemonReload,MemoryMax,MemoryHigh,MemorySwapMax,CPUWeight,CPUQuotaPerSecUSec,IOWeight"
+)
+
+// The jit opt-in (R11, spec §5.11): a second per-start drop-in, by unit
+// name, for every rootfs role. Present only when the root-staged manifest
+// declares jit; removed when the unit stops.
+const (
+ JITDropInName = "50-carlos-jit.conf"
+ JITDropIn = "# Written by carlos instance-control for one start of a release that declares jit (rootfs apps design §5.11); removed when the unit stops.\n[Service]\nMemoryDenyWriteExecute=no\n"
+)
+
+// systemdImageRoot is where systemd mounts a RootImage= while it sets the
+// namespace up; a path it reports under it is a path inside the image.
+const systemdImageRoot = "/run/systemd/mount-rootfs"
+
+// fileIdentity is what the digest cache keys a hash by: the opened file's
+// device, inode, size and mtime, all read from the descriptor that was
+// hashed (fileIdentityOf, linux).
+type fileIdentity struct {
+ dev, ino uint64
+ size int64
+ mtime time.Time
+}
+
+type digestEntry struct {
+ fileIdentity
+ sum string
+}
+
+// imageDigests caches validateStagedImage's hashes by path and identity: a
+// 2 GiB image is hashed once, not on every wake, and any change to the file
+// (a new inode, a new size, a new mtime) hashes it again.
+type imageDigests struct {
+ mu sync.Mutex
+ m map[string]digestEntry
+ hashes int
+}
+
+// digest answers the sha256 of f, opened at path, from the cache when the
+// file's identity is the one that was hashed, and by reading it otherwise.
+func (c *imageDigests) digest(path string, f *os.File) (string, error) {
+ id, err := fileIdentityOf(f)
+ if err != nil {
+ return "", err
+ }
+ c.mu.Lock()
+ e, ok := c.m[path]
+ c.mu.Unlock()
+ if ok && e.fileIdentity == id {
+ return e.sum, nil
+ }
+ h := sha256.New()
+ if _, err := io.Copy(h, f); err != nil {
+ return "", err
+ }
+ sum := hex.EncodeToString(h.Sum(nil))
+ c.mu.Lock()
+ if c.m == nil {
+ c.m = map[string]digestEntry{}
+ }
+ c.m[path] = digestEntry{fileIdentity: id, sum: sum}
+ c.hashes++
+ c.mu.Unlock()
+ return sum, nil
+}
+
+// hashed is how many times the cache has read a file.
+func (c *imageDigests) hashed() int {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ return c.hashes
+}
+
+// validateStagedImage is root's check before it mounts an image into a
+// tenant's namespace (C3, C4). It accepts only what is provably the host's
+// own rootfs release:
+// - the host's row is a rootfs row (HostRuntime), so a binary row can
+// never be started from an image;
+// - the path is exactly release.VersionPath for the host's account and
+// app, as the registry records them, at img.Version;
+// - the file is opened beneath the version store with no symlink at any
+// component (instancefs.OpenBeneath), and its sha256 is img.Digest
+// (cached by file identity);
+// - the staged manifest root wrote beside it is a rootfs manifest whose
+// artifact digest is img.Digest, and it declares the process.
+//
+// Nothing a tenant can write is read: the version store and the sibling
+// are root's, and the registry is the agent's. The bytes are always hashed
+// against the manifest's digest; the sibling's recorded digest alone is
+// never trusted (Task 3's carry: a reused sibling may name a digest the
+// staged bytes do not have). It returns the root-resolved launch.
+func (s *Server) validateStagedImage(host string, role Role, img StagedImage) (RootfsLaunch, error) {
+ if s.HostApp == nil || s.HostRuntime == nil || s.OptDir == "" {
+ return RootfsLaunch{}, fmt.Errorf("%w: this service has no registry view", ErrStagedImage)
+ }
+ rt, err := s.HostRuntime(host)
+ if err != nil {
+ return RootfsLaunch{}, fmt.Errorf("%w: %s has no row on this box (%v)", ErrStagedImage, host, err)
+ }
+ if rt != registry.RuntimeRootfsV1 {
+ return RootfsLaunch{}, fmt.Errorf("%w: %s is not a rootfs instance on this box", ErrStagedImage, host)
+ }
+ if !release.ValidVersion(img.Version) {
+ return RootfsLaunch{}, fmt.Errorf("%w: version %q", ErrStagedImage, img.Version)
+ }
+ if raw, err := hex.DecodeString(img.Digest); err != nil || len(raw) != sha256.Size {
+ return RootfsLaunch{}, fmt.Errorf("%w: digest %q is not a sha256", ErrStagedImage, img.Digest)
+ }
+ acct, app, err := s.HostApp(host)
+ if err != nil || app == "" {
+ return RootfsLaunch{}, fmt.Errorf("%w: %s has no app on this box (%v)", ErrStagedImage, host, err)
+ }
+ acct = release.RouteAccount(registry.Route{Account: acct})
+ want := release.VersionPath(s.OptDir, acct, img.Version, app)
+ if img.Path != want {
+ return RootfsLaunch{}, fmt.Errorf("%w: %q is not %s's staged %s (%q)", ErrStagedImage, img.Path, host, img.Version, want)
+ }
+ versions := filepath.Join(s.OptDir, "versions")
+ rel, err := filepath.Rel(versions, want)
+ if err != nil {
+ return RootfsLaunch{}, fmt.Errorf("%w: %v", ErrStagedImage, err)
+ }
+ f, err := instancefs.OpenBeneath(versions, rel, os.O_RDONLY, 0)
+ if err != nil {
+ return RootfsLaunch{}, fmt.Errorf("%w: opening %s beneath the version store, with no symlink in its path: %v", ErrStagedImage, want, err)
+ }
+ defer f.Close()
+ sum, err := s.imageDigests.digest(want, f)
+ if err != nil {
+ return RootfsLaunch{}, fmt.Errorf("%w: %v", ErrStagedImage, err)
+ }
+ if !strings.EqualFold(sum, img.Digest) {
+ return RootfsLaunch{}, fmt.Errorf("%w: %s has digest %s, the agent verified %s", ErrStagedImage, want, sum, img.Digest)
+ }
+ m, err := release.ReadStagedManifest(s.OptDir, acct, img.Version, app)
+ if err != nil {
+ return RootfsLaunch{}, fmt.Errorf("%w: staged manifest: %v", ErrStagedImage, err)
+ }
+ if m == nil {
+ return RootfsLaunch{}, fmt.Errorf("%w: no staged manifest beside %s", ErrStagedImage, want)
+ }
+ return resolveRootfsLaunch(m, host, role, img)
+}
+
+// resolveRootfsLaunch is the command half of validateStagedImage: from the
+// root-staged manifest m, the launch for role. The run role is the web
+// process; a worker is its own declared process; a cmd is a declared
+// process other than web (the release process for Plan 3, a worker for a
+// one-off run), or an operator's argv (carlos run), which runs where and
+// with what the web process runs (Astra R1 finding 7), so a relative
+// command resolves against the image's working_dir and along its PATH.
+// jit comes from this manifest and nowhere else (spec §5.11).
+func resolveRootfsLaunch(m *release.Manifest, host string, role Role, img StagedImage) (RootfsLaunch, error) {
+ if !m.IsRootfs() {
+ return RootfsLaunch{}, fmt.Errorf("%w: %s", ErrStagedImage, rootfsNotRootfsText)
+ }
+ art, ok := m.BoxArtifact(boxPlatform())
+ if !ok {
+ return RootfsLaunch{}, fmt.Errorf("%w: the staged manifest names no image for this box's platform", ErrStagedImage)
+ }
+ if !strings.EqualFold(art.SHA256, img.Digest) {
+ return RootfsLaunch{}, fmt.Errorf("%w: the staged manifest names digest %s, the image is %s", ErrStagedImage, art.SHA256, img.Digest)
+ }
+ launch := RootfsLaunch{Schema: RootfsLaunchSchema, Host: host, Version: img.Version, Digest: strings.ToLower(img.Digest), JIT: m.JIT}
+ declared := func(name string) (release.Process, error) {
+ p, ok := m.Processes[name]
+ if !ok {
+ return p, fmt.Errorf("%w: "+rootfsUndeclaredText, ErrStagedImage, name)
+ }
+ if len(p.Command) == 0 || p.Command[0] == "" {
+ return p, fmt.Errorf("%w: the %s process declares no command", ErrStagedImage, name)
+ }
+ return p, nil
+ }
+ fill := func(name string, p release.Process) {
+ launch.Process = name
+ launch.Argv = slices.Clone(p.Command)
+ launch.Env = maps.Clone(p.Env)
+ launch.WorkingDir = p.WorkingDir
+ }
+ switch role {
+ case RoleRun:
+ if img.Process == "" || img.Argv != nil {
+ return RootfsLaunch{}, fmt.Errorf("%w: the run role is web, not %q", ErrStagedImage, img.Process)
+ }
+ // An undeclared process is named as such (RFS-2-C08) before the
+ // role rule, so the agent's event line says what the release lacks.
+ p, err := declared(img.Process)
+ if err != nil {
+ return RootfsLaunch{}, err
+ }
+ if img.Process != release.ProcessWeb {
+ return RootfsLaunch{}, fmt.Errorf("%w: the run role is web, not %q", ErrStagedImage, img.Process)
+ }
+ fill(release.ProcessWeb, p)
+ case RoleWorker:
+ if !release.ValidWorkerName(img.Process) || img.Argv != nil {
+ return RootfsLaunch{}, fmt.Errorf("%w: %q is not a worker name", ErrStagedImage, img.Process)
+ }
+ p, err := declared(img.Process)
+ if err != nil {
+ return RootfsLaunch{}, err
+ }
+ fill(img.Process, p)
+ case RoleCmd:
+ switch {
+ case img.Process != "" && len(img.Argv) > 0:
+ return RootfsLaunch{}, fmt.Errorf("%w: a cmd names a process or carries an argv, not both", ErrStagedImage)
+ case img.Process != "":
+ if img.Process == release.ProcessWeb {
+ return RootfsLaunch{}, fmt.Errorf("%w: web is the run role's process, not a cmd's", ErrStagedImage)
+ }
+ p, err := declared(img.Process)
+ if err != nil {
+ return RootfsLaunch{}, err
+ }
+ fill(img.Process, p)
+ case len(img.Argv) > 0:
+ web, err := declared(release.ProcessWeb)
+ if err != nil {
+ return RootfsLaunch{}, err
+ }
+ launch.Process = string(RoleCmd)
+ launch.Argv = slices.Clone(img.Argv)
+ launch.Env = maps.Clone(web.Env)
+ launch.WorkingDir = web.WorkingDir
+ default:
+ return RootfsLaunch{}, fmt.Errorf("%w: a cmd names a process or carries an argv", ErrStagedImage)
+ }
+ default:
+ return RootfsLaunch{}, fmt.Errorf("%w: role %s has no rootfs unit", ErrStagedImage, role)
+ }
+ return launch, nil
+}
+
+// ensureRootfsExe lays out <ExeRoot>/<key>/rootfs/<role>/, the three
+// sources a rootfs unit binds: image (a link to the validated staged
+// image), launch.json (root's resolved launch, 0644, written whole and
+// renamed into place) and resolv.conf (a copy of the resolver file an
+// instance sees). key is the host's instance ID (run, cmd), a GenKey or a
+// WorkerKey. Only root writes under ExeRoot.
+func (s *Server) ensureRootfsExe(key string, role Role, imagePath string, launch RootfsLaunch) error {
+ if s.ExeRoot == "" {
+ return errors.New("no exe root")
+ }
+ if key == "" || strings.ContainsAny(key, "/\x00") || key == "." || key == ".." {
+ return fmt.Errorf("exe: %q is not a key", key)
+ }
+ if role != RoleRun && role != RoleWorker && role != RoleCmd {
+ return fmt.Errorf("exe: role %s has no rootfs layout", role)
+ }
+ dir := RootfsExeDir(s.ExeRoot, key, role)
+ for _, d := range []string{s.ExeRoot, filepath.Join(s.ExeRoot, key), filepath.Dir(dir), dir} {
+ if err := os.Mkdir(d, 0o755); err != nil && !errors.Is(err, os.ErrExist) {
+ return err
+ }
+ fi, err := os.Lstat(d)
+ if err != nil {
+ return err
+ }
+ if !fi.IsDir() {
+ return fmt.Errorf("exe: %s is not a directory", d)
+ }
+ }
+ if err := replaceLink(filepath.Join(dir, RootfsExeImage), imagePath); err != nil {
+ return err
+ }
+ raw, err := json.Marshal(launch)
+ if err != nil {
+ return err
+ }
+ if err := writeFileExcl(filepath.Join(dir, RootfsExeLaunch), append(raw, '\n'), 0o644); err != nil {
+ return err
+ }
+ resolv, err := os.ReadFile(s.resolverFile())
+ if err != nil {
+ return fmt.Errorf("resolv.conf: %w", err)
+ }
+ return writeFileExcl(filepath.Join(dir, RootfsExeResolv), resolv, 0o644)
+}
+
+// writeFileExcl writes data to a fresh temp file beside path (O_EXCL, so a
+// link planted there is never followed) and renames it into place.
+func writeFileExcl(path string, data []byte, mode os.FileMode) error {
+ tmp := path + ".tmp"
+ _ = os.Remove(tmp)
+ f, err := os.OpenFile(tmp, os.O_WRONLY|os.O_CREATE|os.O_EXCL, mode)
+ if err != nil {
+ return err
+ }
+ if _, err := f.Write(data); err != nil {
+ f.Close()
+ os.Remove(tmp)
+ return err
+ }
+ if err := f.Chmod(mode); err != nil {
+ f.Close()
+ os.Remove(tmp)
+ return err
+ }
+ if err := f.Close(); err != nil {
+ os.Remove(tmp)
+ return err
+ }
+ if err := os.Rename(tmp, path); err != nil {
+ os.Remove(tmp)
+ return err
+ }
+ return nil
+}
+
+// removeRootfsExe removes one role's rootfs layout under key.
+func (s *Server) removeRootfsExe(key string, role Role) {
+ if s.ExeRoot == "" || key == "" || strings.ContainsAny(key, "/\x00") || key == "." || key == ".." {
+ return
+ }
+ _ = os.RemoveAll(RootfsExeDir(s.ExeRoot, key, role))
+}
+
+// removeWorkerFiles removes a worker unit's drop-in directory (its own
+// drop-in and any jit one) and its exe directory, by the unit's name as
+// systemd reports it, as removeGenFiles does for a generation.
+func (s *Server) removeWorkerFiles(unit string) error {
+ esc, name, ok := ParseWorkerUnitName(unit)
+ if !ok {
+ return nil
+ }
+ s.removeJITDropIn(unit)
+ var errs []error
+ if s.DropInDir != "" {
+ errs = append(errs, os.RemoveAll(filepath.Join(s.DropInDir, unit+".d")))
+ }
+ iid, ok := Unescape(esc)
+ if !ok || !validInstanceIDPart(iid) {
+ return errors.Join(errs...)
+ }
+ if s.ExeRoot != "" {
+ errs = append(errs, os.RemoveAll(filepath.Join(s.ExeRoot, WorkerKeyOf(iid, name))))
+ }
+ return errors.Join(errs...)
+}
+
+// removeRootfsFiles removes what a stopped rootfs unit leaves, by its kind:
+// a generation's or a worker's files, a cmd's layout, and in every case
+// the unit's jit drop-in. The run unit's standing layout stays, as the
+// binary host's exe layout does.
+func (s *Server) removeRootfsFiles(unit string) {
+ switch {
+ case strings.HasPrefix(unit, UnitPrefixRootfsGen):
+ _ = s.removeGenFiles(unit)
+ case strings.HasPrefix(unit, UnitPrefixRootfsWorker):
+ _ = s.removeWorkerFiles(unit)
+ case strings.HasPrefix(unit, UnitPrefixRootfsCmd):
+ s.removeJITDropIn(unit)
+ if _, esc, ok := ParseUnitName(unit); ok {
+ if iid, ok := Unescape(esc); ok {
+ s.removeRootfsExe(iid, RoleCmd)
+ }
+ }
+ case strings.HasPrefix(unit, UnitPrefixRootfs):
+ s.removeJITDropIn(unit)
+ }
+}
+
+// RootfsGenDropIn is GenDropIn for a rootfs generation: the same header and
+// memory lines, with the image and root's two files in place of the app
+// and live binds (rootfs apps plan 2). The tier is this start's, every
+// time, so a size set later applies at the next start.
+func RootfsGenDropIn(host string, n uint64, tier cgroup.Limit) string {
+ return genDropInHeader(host) + rootfsExeLines(GenKey(host, n), RoleRun) +
+ "StandardOutput=file:" + GenLogSocket(host, n) + "\n" + tierLines(tier)
+}
+
+// WorkerDropIn is the per-start drop-in of a rootfs worker: exactly one
+// Slice= line (the per-host slice), the instance's own two directories,
+// the image and root's two files, and the worker's log socket. No memory
+// lines: a worker shares the host slice's tier (spec §5.4), which is why
+// the tier it is handed is not written.
+func WorkerDropIn(host, name string, _ cgroup.Limit) string {
+ return "# Written by carlos instance-control for one rootfs worker; removed when it stops (rootfs apps plan 2).\n" +
+ "[Service]\n" + hostMountLines(host) + rootfsExeLines(WorkerKey(host, name), RoleWorker) +
+ "StandardOutput=file:" + WorkerLogSocket(host, name) + "\n"
+}
+
+// rootfsExeLines are the RootImage= and BindReadOnlyPaths= lines of a
+// rootfs drop-in for the layout under <exe root>/<key>/rootfs/<role>/.
+func rootfsExeLines(key string, role Role) string {
+ dir := exeRoot + key + "/" + RootfsExeDirName + "/" + string(role) + "/"
+ return "RootImage=" + rootfsRootImage(key, role) + "\n" +
+ "BindReadOnlyPaths=" + dir + RootfsExeLaunch + ":" + RootfsLaunchPath + " " + dir + RootfsExeResolv + ":/etc/resolv.conf\n"
+}
+
+// rootfsRootImage is the RootImage= a rootfs drop-in names.
+func rootfsRootImage(key string, role Role) string {
+ return exeRoot + key + "/" + RootfsExeDirName + "/" + string(role) + "/" + RootfsExeImage
+}
+
+// rootfsView is genView for a rootfs gen or worker unit: one show of the
+// unit and its host slice. applied reports whether systemd's view of the
+// unit is the drop-in just written: its slice, its RootImage (a gen or
+// worker template has none without the drop-in, Task 4's carry, so a
+// start on a stale view would mount nothing), no reload pending, and for a
+// gen (withTier) the tier's memory lines. tierSet reports whether the
+// slice already carries every value setTier would give it. Any failure to
+// answer is a no to both.
+func (s *Server) rootfsView(ctx context.Context, host, unit, image string, tier cgroup.Limit, withTier bool) (slice string, applied, tierSet bool) {
+ out, err := s.Systemctl(ctx, "show", "-p", strings.TrimPrefix(rootfsViewShow, "show -p "), unit, SliceName(host))
+ if err != nil {
+ return "", false, false
+ }
+ blocks := strings.SplitN(strings.TrimLeft(out, "\n"), "\n\n", 2)
+ m := parseShow(blocks[0])
+ applied = m["Slice"] == SliceName(host) && m["RootImage"] == image && m["NeedDaemonReload"] == "no"
+ if withTier {
+ max := strconv.FormatInt(tier.MemoryMax, 10)
+ applied = applied && m["MemoryMax"] == max && m["MemorySwapMax"] == max && m["MemoryHigh"] == strconv.FormatInt(cgroup.HighOf(tier.MemoryMax), 10)
+ }
+ if len(blocks) == 2 {
+ tierSet = sliceAtTier(parseShow(blocks[1]), tier)
+ }
+ return m["Slice"], applied, tierSet
+}
+
+// writeJITDropIn writes the unit's jit drop-in when jit and removes it
+// otherwise, reporting whether the file's presence or bytes changed. A
+// removal counts as owed (removeJITDropIn), so a reload that then fails
+// leaves the next start a reload all the same.
+func (s *Server) writeJITDropIn(unit string, jit bool) (changed bool, err error) {
+ if !jit {
+ return s.removeJITDropIn(unit), nil
+ }
+ return s.writeDropInIfChanged(unit, JITDropInName, JITDropIn)
+}
+
+// removeJITDropIn removes the unit's jit drop-in (stop, sweep, cleanCmdExe,
+// the mismatch path), never the directory a gen or worker drop-in shares.
+// It does not daemon-reload, but a removal that deleted a file marks a
+// reload owed, so the next rootfs start of ANY unit reloads before it
+// starts: systemd may still hold the unit loaded with
+// MemoryDenyWriteExecute=no (systemd.unit(5): a loaded unit's
+// configuration changes only on reload). It reports whether it deleted.
+func (s *Server) removeJITDropIn(unit string) bool {
+ if s.DropInDir == "" || unit == "" {
+ return false
+ }
+ if err := os.Remove(filepath.Join(s.DropInDir, unit+".d", JITDropInName)); err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ s.logf("instance-control: %s: jit drop-in: %v", unit, err)
+ }
+ return false
+ }
+ s.markJITReloadOwed()
+ return true
+}
+
+// markJITReloadOwed records that systemd may hold a unit loaded with a jit
+// setting its files no longer say.
+func (s *Server) markJITReloadOwed() {
+ s.mu.Lock()
+ s.jitRemovals++
+ s.mu.Unlock()
+}
+
+// jitReloadOwed reports whether the next rootfs start must daemon-reload
+// first: this service has never reloaded (a restarted service cannot know
+// what systemd has loaded), or a removal has landed since the last
+// successful reload began.
+func (s *Server) jitReloadOwed() bool {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ return !s.jitReloadedOnce || s.jitRemovals != s.jitReloadedThrough
+}
+
+// jitDaemonReload reloads systemd and, on success, records the removals
+// the reload began after as reloaded. A removal that lands during the
+// reload stays owed.
+func (s *Server) jitDaemonReload(ctx context.Context) error {
+ s.mu.Lock()
+ snap := s.jitRemovals
+ s.mu.Unlock()
+ if out, err := s.Systemctl(ctx, "daemon-reload"); err != nil {
+ return errors.New("daemon-reload: " + strings.TrimSpace(out+" "+err.Error()))
+ }
+ s.mu.Lock()
+ s.jitReloadedThrough, s.jitReloadedOnce = snap, true
+ s.mu.Unlock()
+ return nil
+}
+
+// rootfsUnitOf is the rootfs unit a parsed request names: the worker's,
+// the generation's, or the run or cmd template's. "" for any other role.
+func rootfsUnitOf(req request) string {
+ switch {
+ case req.role == RoleWorker:
+ if req.worker == "" {
+ return ""
+ }
+ return WorkerUnitName(req.host, req.worker)
+ case req.role == RoleRun && req.gen != 0:
+ return RootfsGenUnitName(req.host, req.gen)
+ case req.role == RoleRun, req.role == RoleCmd:
+ return RootfsUnitName(req.role, req.host)
+ }
+ return ""
+}
+
+// rootfsPreflight is what every rootfs launch checks before it lays
+// anything out, in this order: the service has a registry view (403), the
+// box has passed both probes on this boot (503, RFS-2-C02 with the first
+// failing check), and root vouches for the image and resolves the launch
+// (403, validateStagedImage). It reports whether it answered.
+func (s *Server) rootfsPreflight(w http.ResponseWriter, host string, role Role, unit string, img StagedImage) (RootfsLaunch, bool) {
+ if s.HostRuntime == nil {
+ http.Error(w, "not starting "+unit+": "+ErrStagedImage.Error()+": this service has no registry view", http.StatusForbidden)
+ return RootfsLaunch{}, true
+ }
+ if rep, ok := s.ReadyRootfs(); !ok {
+ check := "not yet run"
+ for _, l := range rep.Lines {
+ if l.Verdict == "fail" {
+ check = l.Check
+ break
+ }
+ }
+ http.Error(w, "not starting "+unit+": "+fmt.Sprintf(rootfsProbeNotPassedText, check), http.StatusServiceUnavailable)
+ return RootfsLaunch{}, true
+ }
+ launch, err := s.validateStagedImage(host, role, img)
+ if err != nil {
+ s.logf("instance-control: %s: not starting %s: %v", host, unit, err)
+ http.Error(w, "not starting "+unit+": "+err.Error(), http.StatusForbidden)
+ return RootfsLaunch{}, true
+ }
+ return launch, false
+}
+
+// startRootfs is /start with runtime=rootfs: the run unit (gen 0), a
+// generation, or a worker, from a staged image root vouches for. The
+// order is the brief's: the registry view, the probes, root's validation,
+// the disk and the exe layout, the drop-ins and at most one reload (only
+// when a view does not apply, a jit file changed, or a reload is owed:
+// never on an ordinary start, wake latency 1 and 2c), the tier, the start,
+// and the post-start check of the unit's placement and its
+// MemoryDenyWriteExecute.
+func (s *Server) startRootfs(w http.ResponseWriter, r *http.Request, req request) {
+ ctx := r.Context()
+ if req.role != RoleRun && req.role != RoleWorker {
+ http.Error(w, "role "+string(req.role)+" has no rootfs launch; the rootfs launch roles are run and worker", http.StatusBadRequest)
+ return
+ }
+ if req.role == RoleWorker && req.image.Process != req.worker {
+ http.Error(w, fmt.Sprintf("worker %s names process %q; a worker runs its own process", req.worker, req.image.Process), http.StatusBadRequest)
+ return
+ }
+ unit := rootfsUnitOf(req)
+ if s.DropInDir == "" || s.ExeRoot == "" {
+ http.Error(w, "not starting "+unit+": rootfs units are not configured on this service", http.StatusNotImplemented)
+ return
+ }
+ // The exe key, and what takes the files of a start that fails.
+ var key string
+ var remove func()
+ perStart := req.role == RoleWorker || req.gen != 0
+ switch {
+ case req.role == RoleWorker:
+ key = WorkerKey(req.host, req.worker)
+ remove = func() { _ = s.removeWorkerFiles(unit) }
+ case req.gen != 0:
+ key = GenKey(req.host, req.gen)
+ remove = func() { _ = s.removeGenFiles(unit) }
+ default:
+ key = provision.InstanceID(req.host)
+ remove = func() {
+ s.removeJITDropIn(unit)
+ s.removeRootfsExe(key, RoleRun)
+ }
+ }
+ if req.role == RoleRun && req.gen == 0 {
+ // The host's run unit: an attempt a /stop of this host can cancel
+ // and join (legacystart.go, #292), as for the binary run unit.
+ attempt, finish, err := s.beginLegacyStart(ctx, req.host, requestSeq(ctx))
+ if err != nil {
+ http.Error(w, "start: "+err.Error(), http.StatusConflict)
+ return
+ }
+ defer finish()
+ ctx = attempt
+ }
+ launch, answered := s.rootfsPreflight(w, req.host, req.role, unit, req.image)
+ if answered {
+ return
+ }
+ if perStart {
+ s.markGenStarting(unit, true)
+ defer s.markGenStarting(unit, false)
+ }
+ began := time.Now()
+ if s.refuseStart(w, req.host, unit, func() error {
+ err := s.ensureRootfsExe(key, req.role, req.image.Path, launch)
+ if err != nil {
+ remove()
+ }
+ return err
+ }) {
+ return
+ }
+ fail := func(status int, msg string) {
+ remove()
+ http.Error(w, msg, status)
+ }
+ // The drop-ins. A gen's name is never reused, so its drop-in is written
+ // every time; a worker's is, so its drop-in is written only when the
+ // bytes differ (rewriting moves the mtime and systemd then reports
+ // NeedDaemonReload for a loaded unit). The jit drop-in is written or
+ // removed for every role.
+ var image string
+ withTier := false
+ switch {
+ case req.role == RoleWorker:
+ if _, err := s.writeDropInIfChanged(unit, DropInName, WorkerDropIn(req.host, req.worker, req.tier)); err != nil {
+ fail(http.StatusBadGateway, "drop-in: "+err.Error())
+ return
+ }
+ image = rootfsRootImage(key, RoleWorker)
+ case req.gen != 0:
+ if err := s.writeDropIn(unit, DropInName, RootfsGenDropIn(req.host, req.gen, req.tier)); err != nil {
+ fail(http.StatusBadGateway, "drop-in: "+err.Error())
+ return
+ }
+ image, withTier = rootfsRootImage(key, RoleRun), true
+ }
+ changed, err := s.writeJITDropIn(unit, launch.JIT)
+ if err != nil {
+ fail(http.StatusBadGateway, "jit drop-in: "+err.Error())
+ return
+ }
+ var slice string
+ applied, tierSet := true, false
+ if perStart {
+ slice, applied, tierSet = s.rootfsView(ctx, req.host, unit, image, req.tier, withTier)
+ }
+ reload := "skipped"
+ if !applied || changed || s.jitReloadOwed() {
+ reloadAt := time.Now()
+ if err := s.jitDaemonReload(ctx); err != nil {
+ fail(http.StatusBadGateway, err.Error())
+ return
+ }
+ if perStart {
+ if slice, applied, tierSet = s.rootfsView(ctx, req.host, unit, image, req.tier, withTier); !applied {
+ fail(http.StatusBadGateway, "not starting "+unit+": its drop-in did not apply (slice "+strconv.Quote(slice)+")")
+ return
+ }
+ }
+ reload = time.Since(reloadAt).Round(time.Millisecond).String()
+ }
+ // The tier (Astra R1 finding 5): a gen-0 run start sets the host
+ // slice's tier, as a cold instance's start does. A generation or a
+ // worker leaves a populated slice alone, as startGen does: it may carry
+ // the swap's raised limit for two generations.
+ if req.role == RoleRun && req.gen == 0 || !s.slicePopulated(req.host) && !tierSet {
+ if err := s.setTier(ctx, req.host, req.tier); err != nil {
+ fail(http.StatusBadGateway, "set-property: "+err.Error())
+ return
+ }
+ }
+ // #328: s.Egress.Assert(ctx, host) here (egress checked before the unit starts)
+ if err := ctx.Err(); err != nil {
+ fail(http.StatusConflict, "start: cancelled before it was submitted: "+err.Error())
+ return
+ }
+ startAt := time.Now()
+ if out, err := s.Systemctl(ctx, "start", unit); err != nil {
+ msg := rootfsStartFailure(unit, out, err)
+ if perStart {
+ s.abandonUnit(ctx, w, req.host, unit, msg, remove)
+ return
+ }
+ http.Error(w, msg, http.StatusBadGateway)
+ return
+ }
+ inv, ok := s.rootfsPlaced(ctx, w, req.host, unit, launch.JIT, remove)
+ if !ok {
+ return
+ }
+ s.logf("instance-control: %s: started %s from %s memory.max=%d (prepared in %s, reload %s, start %s)", req.host, unit, launch.Version, req.tier.MemoryMax,
+ startAt.Sub(began).Round(time.Millisecond), reload, time.Since(startAt).Round(time.Millisecond))
+ writeJSON(w, http.StatusOK, StartResponse{InvocationID: inv})
+}
+
+// rootfsStartFailure is the 502 text of a failed systemctl start: when
+// systemd names a path the image lacks, RFS-2-C10 with that path, so the
+// agent's event line says which mount point to add; otherwise systemd's
+// own output (a missing bind source under the exe root is the box's fault,
+// not the image's, preflight 2.14).
+func rootfsStartFailure(unit, out string, err error) string {
+ if p, ok := rootfsMountFailure(out); ok {
+ return "not starting " + unit + ": " + fmt.Sprintf(rootfsMissingPathText, p)
+ }
+ return "start: " + strings.TrimSpace(out+" "+err.Error())
+}
+
+// rootfsMountFailure finds, in systemctl's output, a path under systemd's
+// image root that did not exist (the ENOENT of a mount point the image
+// lacks), and answers the path inside the image. Any other path, and any
+// other error, is not the image's.
+func rootfsMountFailure(out string) (string, bool) {
+ for _, line := range strings.Split(out, "\n") {
+ if !strings.Contains(line, "No such file or directory") {
+ continue
+ }
+ i := strings.Index(line, systemdImageRoot+"/")
+ if i < 0 {
+ continue
+ }
+ rest := line[i+len(systemdImageRoot):]
+ if j := strings.IndexByte(rest, ':'); j >= 0 {
+ rest = rest[:j]
+ }
+ p := strings.TrimSpace(rest)
+ if p == "" || p == "/" || path.Clean(p) != p {
+ continue
+ }
+ return p, true
+ }
+ return "", false
+}
+
+// rootfsPlaced is the post-start check of every rootfs start (C5, C10): one
+// show of the unit's invocation, slice, control group and
+// MemoryDenyWriteExecute. The control group must be the unit's own under
+// the host slice (a unit that has already exited has none, so its
+// configured Slice stands in); MemoryDenyWriteExecute must be no when the
+// release declares jit and yes otherwise. On a mismatch the unit is
+// stopped and waited on, its jit drop-in removed, a reload marked owed
+// (the loaded configuration was stale even when no file was there), its
+// files taken, and the request answered 502 naming the property. ok is
+// false when it answered.
+func (s *Server) rootfsPlaced(ctx context.Context, w http.ResponseWriter, host, unit string, jit bool, remove func()) (inv string, ok bool) {
+ end := func(msg string) (string, bool) {
+ s.endMisplaced(ctx, host, unit, remove)
+ s.logf("instance-control: %s: %s", host, msg)
+ http.Error(w, msg, http.StatusBadGateway)
+ return "", false
+ }
+ out, err := s.Systemctl(ctx, "show", "-p", strings.TrimPrefix(rootfsPostStartShow, "show -p "), unit)
+ if err != nil {
+ return end("show: " + err.Error())
+ }
+ m := parseShow(out)
+ slice := SliceName(host)
+ own := "/" + parentSlices + "/" + slice + "/" + unit
+ if cg := m["ControlGroup"]; cg != "" && cg != own || cg == "" && m["Slice"] != slice {
+ return end("not serving " + unit + ": its control group is " + strconv.Quote(cg) + " in slice " + strconv.Quote(m["Slice"]) + ", not " + own)
+ }
+ want := "yes"
+ if jit {
+ want = "no"
+ }
+ if got := m["MemoryDenyWriteExecute"]; got != want {
+ return end(fmt.Sprintf("not serving %s: MemoryDenyWriteExecute is %q, the release's jit is %v", unit, got, jit))
+ }
+ return m["InvocationID"], true
+}
+
+// endMisplaced stops a started unit the post-start check refused, on a
+// context the client's hang-up cannot cancel, waits for its cgroup, drops
+// its jit drop-in, marks the reload owed and takes its files.
+func (s *Server) endMisplaced(ctx context.Context, host, unit string, remove func()) {
+ dctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), DeadlineControl+s.waitBudget())
+ defer cancel()
+ s.stopUnit(dctx, unit)
+ wctx, waitCancel := context.WithTimeout(dctx, s.waitBudget())
+ defer waitCancel()
+ if err := WaitCgroupEmpty(wctx, s.CgroupRoot, SliceName(host), unit, 50*time.Millisecond); err != nil {
+ s.logf("instance-control: %s: %s: %v", host, unit, err)
+ }
+ s.removeJITDropIn(unit)
+ s.markJITReloadOwed()
+ remove()
+}
+
+// startRootfsCmd is handleCmd's rootfs branch (run spec §5.4 for a rootfs
+// row), on the rootfs cmd unit: the per-host cmd lock, the loaded-state
+// refusal and reset-failed, the rootfs preflight, the layout, the jit
+// drop-in and its reload, the tier rule of a populated slice, start
+// --no-block, the invocation poll, then the post-start check on the new
+// invocation.
+func (s *Server) startRootfsCmd(w http.ResponseWriter, r *http.Request, req request) {
+ lock, _ := s.cmdLocks.LoadOrStore(req.host, new(sync.Mutex))
+ lock.(*sync.Mutex).Lock()
+ defer lock.(*sync.Mutex).Unlock()
+ ctx := r.Context()
+ unit := RootfsUnitName(RoleCmd, req.host)
+ if s.DropInDir == "" || s.ExeRoot == "" {
+ http.Error(w, "not starting "+unit+": rootfs units are not configured on this service", http.StatusNotImplemented)
+ return
+ }
+ switch state := s.showValue(ctx, "ActiveState", unit); state {
+ case "active", "activating", "deactivating", "reloading", "refreshing":
+ http.Error(w, "cmd: "+unit+" is "+state, http.StatusConflict)
+ return
+ case "failed":
+ if out, err := s.Systemctl(ctx, "reset-failed", unit); err != nil {
+ http.Error(w, "reset-failed: "+strings.TrimSpace(out+" "+err.Error()), http.StatusBadGateway)
+ return
+ }
+ }
+ launch, answered := s.rootfsPreflight(w, req.host, RoleCmd, unit, req.image)
+ if answered {
+ return
+ }
+ key := provision.InstanceID(req.host)
+ remove := func() {
+ s.removeJITDropIn(unit)
+ s.removeRootfsExe(key, RoleCmd)
+ }
+ if s.refuseStart(w, req.host, unit, func() error {
+ err := s.ensureRootfsExe(key, RoleCmd, req.image.Path, launch)
+ if err != nil {
+ remove()
+ }
+ return err
+ }) {
+ return
+ }
+ fail := func(status int, msg string) {
+ remove()
+ http.Error(w, msg, status)
+ }
+ changed, err := s.writeJITDropIn(unit, launch.JIT)
+ if err != nil {
+ fail(http.StatusBadGateway, "jit drop-in: "+err.Error())
+ return
+ }
+ if changed || s.jitReloadOwed() {
+ if err := s.jitDaemonReload(ctx); err != nil {
+ fail(http.StatusBadGateway, err.Error())
+ return
+ }
+ }
+ // A command runs beside an awake instance, whose start set the slice;
+ // a populated slice is left alone, as handleCmd leaves it.
+ if s.slicePopulated(req.host) {
+ s.logf("instance-control: %s: slice populated, leaving its limit to the instance's owner", req.host)
+ } else if err := s.setTier(ctx, req.host, req.tier); err != nil {
+ fail(http.StatusBadGateway, "set-property: "+err.Error())
+ return
+ }
+ // #328: s.Egress.Assert(ctx, host) here (egress checked before the unit starts)
+ before := s.showValue(ctx, "InvocationID", unit)
+ if out, err := s.Systemctl(ctx, "start", "--no-block", unit); err != nil {
+ fail(http.StatusBadGateway, rootfsStartFailure(unit, out, err))
+ return
+ }
+ deadline := time.Now().Add(s.cmdStartWait())
+ for {
+ if inv := s.showValue(ctx, "InvocationID", unit); inv != "" && inv != before {
+ if _, ok := s.rootfsPlaced(ctx, w, req.host, unit, launch.JIT, remove); !ok {
+ return
+ }
+ s.logf("instance-control: %s: started %s (%s) from %s memory.max=%d", req.host, unit, launch.Process, launch.Version, req.tier.MemoryMax)
+ writeJSON(w, http.StatusOK, StartResponse{InvocationID: inv})
+ return
+ }
+ if time.Now().After(deadline) || ctx.Err() != nil {
+ dctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 30*time.Second)
+ s.stopUnit(dctx, unit)
+ cancel()
+ fail(http.StatusBadGateway, "cmd: "+unit+" showed no new invocation")
+ return
+ }
+ time.Sleep(s.cmdPoll())
+ }
+}
+
+// stopRootfs is /stop with runtime=rootfs: the run unit (with the legacy
+// start fence, as the binary run unit), a generation (kill=1 kills it
+// outright), a worker, or the cmd unit. A stopped unit's files go with it:
+// a generation's or worker's drop-in directory and exe layout, a cmd's
+// layout, and in every case the unit's jit drop-in.
+func (s *Server) stopRootfs(w http.ResponseWriter, r *http.Request, req request) {
+ ctx := r.Context()
+ unit := rootfsUnitOf(req)
+ if unit == "" {
+ http.Error(w, "role "+string(req.role)+" has no rootfs unit", http.StatusBadRequest)
+ return
+ }
+ if req.role == RoleRun && req.gen == 0 {
+ release, err := s.fenceLegacyStarts(ctx, req.host)
+ defer release()
+ if err != nil {
+ _, _ = s.Systemctl(ctx, "stop", unit)
+ http.Error(w, "stop: "+err.Error(), http.StatusBadGateway)
+ return
+ }
+ }
+ if req.kill {
+ if err := s.killGen(ctx, req.host, unit); err != nil {
+ http.Error(w, err.Error(), http.StatusBadGateway)
+ return
+ }
+ } else if out, err := s.Systemctl(ctx, "stop", unit); err != nil {
+ http.Error(w, "stop: "+strings.TrimSpace(out+" "+err.Error()), http.StatusBadGateway)
+ return
+ }
+ if req.gen != 0 {
+ s.resetGenFailed(ctx, unit)
+ }
+ if req.role == RoleCmd {
+ s.cleanCmdExe(ctx, req.host, true)
+ } else {
+ s.removeRootfsFiles(unit)
+ }
+ w.WriteHeader(http.StatusNoContent)
+}
+
+// setRootfsReport stores the rootfs probe's verdict.
+func (s *Server) setRootfsReport(r ProbeReport) {
+ s.mu.Lock()
+ s.rootfsReport, s.rootfsProbed = r, true
+ s.mu.Unlock()
+}
+
+// runRootfsProbe runs the rootfs probe after the binary one. Task 7 fills
+// it in; until then it records that the probe has not run, so no box is
+// rootfs-ready on this build.
+func (s *Server) runRootfsProbe(ctx context.Context) {
+ if s.ProbeImage == nil {
+ s.setRootfsReport(ProbeReport{Lines: []VerdictLine{{Check: "rootfs-probe", Verdict: "unverified", Detail: "no probe image on this service"}}})
+ return
+ }
+ s.setRootfsReport(ProbeReport{Lines: []VerdictLine{{Check: "rootfs-probe", Verdict: "fail", Detail: "not yet run"}}})
+}
+
+// ReadyRootfs is the rootfs verdict: the binary probe's report is ready
+// (the box is nested-ready, and this process runs the installed binary),
+// and the rootfs probe's is. The report is the rootfs one; when the binary
+// probe is the reason, a synthetic nested line naming its first failing
+// check is put first.
+func (s *Server) ReadyRootfs() (ProbeReport, bool) {
+ binary, nested := s.Ready()
+ s.mu.Lock()
+ rep, probed := s.rootfsReport, s.rootfsProbed
+ s.mu.Unlock()
+ if !probed {
+ rep = ProbeReport{Lines: []VerdictLine{{Check: "rootfs-probe", Verdict: "fail", Detail: "not yet run"}}}
+ }
+ // A copy: the stored report must not grow a line per call.
+ out := ProbeReport{Ready: rep.Ready, Lines: append([]VerdictLine(nil), rep.Lines...)}
+ if !nested {
+ detail := "not ready"
+ for _, l := range binary.Lines {
+ if l.Verdict == "fail" {
+ detail = l.Check + " " + l.Verdict + " " + l.Detail
+ break
+ }
+ }
+ out.Ready = false
+ out.Lines = append([]VerdictLine{{Check: "nested", Verdict: "fail", Detail: strings.TrimSpace(detail)}}, out.Lines...)
+ }
+ return out, out.Ready
+}
+
+// handleReadyRootfs is GET /ready/rootfs, handleReady's shape with the
+// rootfs verdict.
+func (s *Server) handleReadyRootfs(w http.ResponseWriter, r *http.Request) {
+ if s.Build != "" {
+ w.Header().Set(BuildHeader, s.Build)
+ }
+ rep, ready := s.ReadyRootfs()
+ if ready {
+ w.WriteHeader(http.StatusNoContent)
+ return
+ }
+ writeJSON(w, http.StatusServiceUnavailable, rep)
+}
diff --git a/internal/instancectl/rootfs_identity_linux.go b/internal/instancectl/rootfs_identity_linux.go
new file mode 100644
index 000000000..7c4eb26b6
--- /dev/null
+++ b/internal/instancectl/rootfs_identity_linux.go
@@ -0,0 +1,22 @@
+//go:build linux
+
+package instancectl
+
+import (
+ "os"
+ "time"
+
+ "golang.org/x/sys/unix"
+)
+
+// fileIdentityOf reads the identity of the opened file itself (fstat), so
+// the digest cache keys a hash by the inode that was hashed, never by a
+// path that may since name another file.
+func fileIdentityOf(f *os.File) (fileIdentity, error) {
+ var st unix.Stat_t
+ if err := unix.Fstat(int(f.Fd()), &st); err != nil {
+ return fileIdentity{}, &os.PathError{Op: "fstat", Path: f.Name(), Err: err}
+ }
+ sec, nsec := st.Mtim.Unix()
+ return fileIdentity{dev: uint64(st.Dev), ino: uint64(st.Ino), size: st.Size, mtime: time.Unix(sec, nsec)}, nil
+}
diff --git a/internal/instancectl/rootfs_identity_other.go b/internal/instancectl/rootfs_identity_other.go
new file mode 100644
index 000000000..84139bf54
--- /dev/null
+++ b/internal/instancectl/rootfs_identity_other.go
@@ -0,0 +1,12 @@
+//go:build !linux
+
+package instancectl
+
+import (
+ "errors"
+ "os"
+)
+
+// fileIdentityOf is linux-only: the control service runs there, and a
+// rootfs image is never vouched for anywhere else.
+func fileIdentityOf(*os.File) (fileIdentity, error) { return fileIdentity{}, errors.ErrUnsupported }
diff --git a/internal/instancectl/rootfs_test.go b/internal/instancectl/rootfs_test.go
new file mode 100644
index 000000000..05cd8d3cc
--- /dev/null
+++ b/internal/instancectl/rootfs_test.go
@@ -0,0 +1,1751 @@
+//go:build linux
+
+package instancectl
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "net/netip"
+ "net/url"
+ "os"
+ "path/filepath"
+ "reflect"
+ "strconv"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/cgroup"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/version"
+)
+
+// The rootfs start path's fixture: a staged image and its root-written
+// manifest sibling under a version store, a Server whose registry view says
+// the host is a rootfs row, and a model of systemd behind fakeCtl.dynamic
+// that hands out invocations on start, tracks each unit's ActiveState, and
+// answers the post-start property show for any unit a test did not answer
+// explicitly. Explicit fakeCtl.answers always win over the model.
+
+const (
+ rootfsHost = "shop.example.com"
+ rootfsAcct = "acme"
+ rootfsApp = "shop"
+ rootfsVer = "v1"
+)
+
+// rootfsShared is what a restarted control service keeps: the directories
+// on disk and systemd (the fake and its model).
+type rootfsShared struct {
+ opt, exe, dropins, cgroup, resolv string
+ ctl *fakeCtl
+
+ mu sync.Mutex
+ seq int
+ invocations map[string]string
+ active map[string]bool
+ // loaded is, per unit, whether its jit drop-in existed at the last
+ // daemon-reload; loadedModel makes MemoryDenyWriteExecute follow it
+ // (f.loadedJIT) instead of the file on disk.
+ loaded map[string]bool
+ loadedModel bool
+}
+
+type rootfsBox struct {
+ t *testing.T
+ srv *Server
+ ctl *fakeCtl
+ ts *httptest.Server
+ host string
+ imagePath string
+ image []byte
+ digest string
+ resolvConf string
+ manifest release.Manifest
+ shared *rootfsShared
+}
+
+func rootfsFixture(t *testing.T) *rootfsBox {
+ t.Helper()
+ root := t.TempDir()
+ sh := &rootfsShared{
+ opt: filepath.Join(root, "opt"), exe: filepath.Join(root, "exe"), dropins: filepath.Join(root, "dropins"),
+ cgroup: filepath.Join(root, "cgroup"), resolv: filepath.Join(root, "resolv.conf"),
+ ctl: &fakeCtl{answers: map[string]string{}, fail: map[string]error{}},
+ invocations: map[string]string{}, active: map[string]bool{}, loaded: map[string]bool{},
+ }
+ for _, d := range []string{sh.exe, sh.dropins, sh.cgroup} {
+ if err := os.MkdirAll(d, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := os.WriteFile(sh.resolv, []byte("nameserver 10.0.0.2\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ f := newRootfsBox(t, sh)
+ // The staged image and its root-written sibling.
+ f.image = []byte("squashfs image of shop v1, a few bytes stand in for the real thing")
+ f.imagePath = release.VersionPath(sh.opt, rootfsAcct, rootfsVer, rootfsApp)
+ f.digest = writeStaged(t, f.imagePath, string(f.image))
+ f.manifest = release.Manifest{Schema: 1, App: rootfsApp, Version: rootfsVer, Kind: release.KindRootfs,
+ Artifacts: map[string]release.Artifact{release.RootfsArtifactKey: {File: release.RootfsArtifactFile, SHA256: f.digest, Size: int64(len(f.image))}},
+ Processes: map[string]release.Process{
+ release.ProcessWeb: {Command: []string{"/app/serve"}, WorkingDir: "/app", Env: map[string]string{"PATH": "/app/bin:/usr/local/bundle/bin:/usr/bin", "RAILS_ENV": "production"}},
+ "jobs": {Command: []string{"/app/jobs"}},
+ release.ProcessRelease: {Command: []string{"/app/migrate"}, WorkingDir: "/app", Env: map[string]string{"PATH": "/app/bin:/usr/bin"}},
+ }}
+ f.restageManifest(t, func(*release.Manifest) {})
+ return f
+}
+
+// newRootfsFixture builds the Server over shared, as the first fixture does
+// and as f.restart does for a restarted control service. The box platform
+// is pinned to the one rootfs manifests name.
+func newRootfsBox(t *testing.T, sh *rootfsShared) *rootfsBox {
+ t.Helper()
+ old := boxPlatform
+ boxPlatform = func() string { return "linux-arm64" }
+ t.Cleanup(func() { boxPlatform = old })
+ f := &rootfsBox{t: t, ctl: sh.ctl, host: rootfsHost, shared: sh, resolvConf: "nameserver 10.0.0.2\n"}
+ f.srv = &Server{
+ Systemctl: sh.ctl.run,
+ PeerOK: func(net.Conn) error { return nil },
+ CgroupRoot: sh.cgroup,
+ ProbeOut: filepath.Join(t.TempDir(), "probe.out"),
+ FixtureOK: func() error { return nil },
+ ProbeEvery: time.Hour,
+ Logf: t.Logf,
+ Build: version.Build,
+ ExeRoot: sh.exe,
+ OptDir: sh.opt,
+ DropInDir: sh.dropins,
+ ResolvConf: sh.resolv,
+ HostApp: func(host string) (string, string, error) {
+ if host != rootfsHost {
+ return "", "", errors.New("no such route")
+ }
+ return rootfsAcct, rootfsApp, nil
+ },
+ HostRuntime: func(host string) (string, error) {
+ if host != rootfsHost {
+ return "", errors.New("no such route")
+ }
+ return registry.RuntimeRootfsV1, nil
+ },
+ LocalAddrs: func() ([]netip.Prefix, error) { return []netip.Prefix{netip.MustParsePrefix("172.31.7.19/32")}, nil },
+ }
+ f.srv.cmdPollEvery, f.srv.cmdWaitBound, f.srv.cmdStartBound = 5*time.Millisecond, 300*time.Millisecond, 300*time.Millisecond
+ f.srv.cgroupWaitBudget = time.Second
+ // Both probes have passed on this boot.
+ f.srv.mu.Lock()
+ f.srv.report, f.srv.probed = ProbeReport{Ready: true, Lines: []VerdictLine{{Check: "pid1", Verdict: "ok"}}}, true
+ f.srv.mu.Unlock()
+ f.srv.setRootfsReport(ProbeReport{Ready: true, Lines: []VerdictLine{{Check: "root-image", Verdict: "ok"}}})
+ sh.ctl.mu.Lock()
+ sh.ctl.dynamic = f.model
+ sh.ctl.mu.Unlock()
+ f.ts = httptest.NewServer(f.srv.Handler())
+ t.Cleanup(f.ts.Close)
+ return f
+}
+
+// restart is a new control service over the same disk and the same
+// systemd: it knows nothing of the reloads the old one did.
+func (f *rootfsBox) restart(t *testing.T) *rootfsBox {
+ t.Helper()
+ n := newRootfsBox(t, f.shared)
+ n.image, n.imagePath, n.digest, n.manifest = f.image, f.imagePath, f.digest, f.manifest
+ return n
+}
+
+// model is the fixture's systemd. It defers to an explicit answer when a
+// test set one for the exact call.
+func (f *rootfsBox) model(args []string) (string, bool) {
+ line := strings.Join(args, " ")
+ if f.ctl.hasAnswer(line) {
+ return "", false
+ }
+ sh := f.shared
+ sh.mu.Lock()
+ defer sh.mu.Unlock()
+ unit := args[len(args)-1]
+ switch {
+ case args[0] == "start":
+ sh.seq++
+ sh.invocations[unit] = fmt.Sprintf("%032x", sh.seq)
+ sh.active[unit] = true
+ return "", true
+ case args[0] == "stop":
+ sh.active[unit] = false
+ return "", true
+ case args[0] == "daemon-reload":
+ entries, _ := os.ReadDir(sh.dropins)
+ for _, e := range entries {
+ u, ok := strings.CutSuffix(e.Name(), ".d")
+ if !ok {
+ continue
+ }
+ _, err := os.Stat(filepath.Join(sh.dropins, e.Name(), JITDropInName))
+ sh.loaded[u] = err == nil
+ }
+ return "", true
+ case strings.HasPrefix(line, "show -p InvocationID --value "):
+ return sh.invocations[unit] + "\n", true
+ case strings.HasPrefix(line, "show -p ActiveState --value "):
+ if sh.active[unit] {
+ return "active\n", true
+ }
+ return "inactive\n", true
+ case strings.HasPrefix(line, rootfsPostStartShow+" "):
+ return "InvocationID=" + sh.invocations[unit] + "\nSlice=" + SliceName(f.host) + "\nControlGroup=/carlos.slice/carlos-instances.slice/" + SliceName(f.host) + "/" + unit + "\nMemoryDenyWriteExecute=" + f.mdweLocked(unit) + "\n", true
+ case strings.HasPrefix(line, rootfsViewShow+" "):
+ unit = args[len(args)-2]
+ return rootfsViewAnswer(f.host, unit, true), true
+ }
+ return "", false
+}
+
+// mdweLocked is the model's effective MemoryDenyWriteExecute for unit: with
+// loadedModel, what the unit was loaded with at the last daemon-reload
+// (systemd.unit(5)); otherwise the file on disk right now.
+func (f *rootfsBox) mdweLocked(unit string) string {
+ var jit bool
+ if f.shared.loadedModel {
+ jit = f.shared.loaded[unit]
+ } else {
+ _, err := os.Stat(filepath.Join(f.shared.dropins, unit+".d", JITDropInName))
+ jit = err == nil
+ }
+ if jit {
+ return "no"
+ }
+ return "yes"
+}
+
+// loadedJIT switches the model to systemd's real rule: a loaded unit's
+// configuration changes only on daemon-reload.
+func (f *rootfsBox) loadedJIT(t *testing.T) {
+ t.Helper()
+ f.shared.mu.Lock()
+ f.shared.loadedModel = true
+ f.shared.mu.Unlock()
+}
+
+// rootfsViewAnswer is systemd's answer to the rootfsView show for a gen or
+// worker unit: the unit's block, then the host slice's (which carries no
+// tier, so tierSet is false). applied gives the slice and the RootImage the
+// drop-in names and NeedDaemonReload=no; otherwise the unit sits in -.slice.
+func rootfsViewAnswer(host, unit string, applied bool) string {
+ max := strconv.FormatInt(cgroup.TierChildren.MemoryMax, 10)
+ high := strconv.FormatInt(cgroup.HighOf(cgroup.TierChildren.MemoryMax), 10)
+ tier := "MemoryMax=" + max + "\nMemoryHigh=" + high + "\nMemorySwapMax=" + max + "\n"
+ block := "Slice=-.slice\nRootImage=\nNeedDaemonReload=yes\n" + tier
+ if applied {
+ block = "Slice=" + SliceName(host) + "\nRootImage=" + rootImageOf(unit) + "\nNeedDaemonReload=no\n" + tier
+ }
+ return block + "\nSlice=carlos-instances.slice\nNeedDaemonReload=no\nMemoryMax=infinity\nMemoryHigh=infinity\nMemorySwapMax=infinity\nCPUWeight=100\nCPUQuotaPerSecUSec=infinity\nIOWeight=100\n"
+}
+
+// rootImageOf is the RootImage= a gen or worker drop-in writes for unit.
+func rootImageOf(unit string) string {
+ if esc, n, ok := ParseRootfsGenUnitName(unit); ok {
+ iid, _ := Unescape(esc)
+ return "/run/carlos-control/exe/" + GenKeyOf(iid, n) + "/rootfs/run/image"
+ }
+ if esc, name, ok := ParseWorkerUnitName(unit); ok {
+ iid, _ := Unescape(esc)
+ return "/run/carlos-control/exe/" + WorkerKeyOf(iid, name) + "/rootfs/worker/image"
+ }
+ return ""
+}
+
+// do issues one request to the real handler. A query key "_argv" is the
+// JSON body a /cmd carries (cmdQuery puts it there).
+func (f *rootfsBox) do(t *testing.T, method, path string, q url.Values) (int, string) {
+ t.Helper()
+ var body io.Reader
+ if q != nil {
+ if argv := q.Get("_argv"); argv != "" {
+ body = strings.NewReader(argv)
+ q = cloneValues(q)
+ q.Del("_argv")
+ }
+ }
+ u := f.ts.URL + path
+ if len(q) > 0 {
+ u += "?" + q.Encode()
+ }
+ req, err := http.NewRequest(method, u, body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ resp, err := f.ts.Client().Do(req)
+ if err != nil {
+ t.Fatalf("%s %s: %v", method, path, err)
+ }
+ defer resp.Body.Close()
+ b, _ := io.ReadAll(resp.Body)
+ return resp.StatusCode, strings.TrimSpace(string(b))
+}
+
+func (f *rootfsBox) post(t *testing.T, path string, q url.Values) (int, string) {
+ t.Helper()
+ return f.do(t, http.MethodPost, path, q)
+}
+
+func (f *rootfsBox) get(t *testing.T, path string, q url.Values, out any) int {
+ t.Helper()
+ status, body := f.do(t, http.MethodGet, path, q)
+ if out != nil && status == 200 {
+ if err := json.Unmarshal([]byte(body), out); err != nil {
+ t.Fatalf("GET %s: %v: %s", path, err, body)
+ }
+ }
+ return status
+}
+
+func cloneValues(q url.Values) url.Values {
+ out := url.Values{}
+ for k, v := range q {
+ out[k] = append([]string(nil), v...)
+ }
+ return out
+}
+
+// client is the agent-side Client dialled at the fixture's server.
+func (f *rootfsBox) client(t *testing.T) *Client {
+ t.Helper()
+ addr := f.ts.Listener.Addr().String()
+ return &Client{Path: addr, Dial: func(ctx context.Context, _ string) (net.Conn, error) {
+ return (&net.Dialer{}).DialContext(ctx, "tcp", addr)
+ }}
+}
+
+func (f *rootfsBox) img(process string) StagedImage {
+ return StagedImage{Path: f.imagePath, Version: rootfsVer, Digest: f.digest, Process: process}
+}
+
+func (f *rootfsBox) startQuery(role Role, process string, gen uint64) url.Values {
+ q := url.Values{"host": {f.host}, "role": {string(role)}, "memory_max": {strconv.FormatInt(cgroup.TierChildren.MemoryMax, 10)},
+ "runtime": {"rootfs"}, "image": {f.imagePath}, "version": {rootfsVer}, "digest": {f.digest}, "process": {process}}
+ if gen > 0 {
+ q.Set("gen", strconv.FormatUint(gen, 10))
+ }
+ return q
+}
+
+func (f *rootfsBox) cmdQuery(process string, argv []string) url.Values {
+ q := url.Values{"host": {f.host}, "memory_max": {strconv.FormatInt(cgroup.TierChildren.MemoryMax, 10)},
+ "runtime": {"rootfs"}, "image": {f.imagePath}, "version": {rootfsVer}, "digest": {f.digest}}
+ if process != "" {
+ q.Set("process", process)
+ }
+ if argv != nil {
+ b, _ := json.Marshal(map[string]any{"argv": argv})
+ q.Set("_argv", string(b))
+ }
+ return q
+}
+
+// restageManifest writes the staged sibling from a mutated copy of the
+// fixture's manifest, as root's adopt pass staged it.
+func (f *rootfsBox) restageManifest(t *testing.T, mut func(m *release.Manifest)) {
+ t.Helper()
+ raw, _ := json.Marshal(f.manifest)
+ var m release.Manifest
+ if err := json.Unmarshal(raw, &m); err != nil {
+ t.Fatal(err)
+ }
+ mut(&m)
+ raw, err := json.Marshal(m)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := release.StageManifest(f.shared.opt, rootfsAcct, rootfsVer, rootfsApp, raw); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// symlinkVersionDir puts a symlink on the image's path: versions/v1 becomes
+// a link to the moved directory.
+func (f *rootfsBox) symlinkVersionDir(t *testing.T) {
+ t.Helper()
+ dir := filepath.Join(f.shared.opt, "versions", rootfsVer)
+ if err := os.Rename(dir, dir+".real"); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Symlink(dir+".real", dir); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// rewriteImage replaces the image's bytes in place (same inode and size)
+// and moves its mtime on, so only the cache's mtime check can see it.
+func (f *rootfsBox) rewriteImage(t *testing.T, data []byte) {
+ t.Helper()
+ if err := os.WriteFile(f.imagePath, data, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ later := time.Now().Add(2 * time.Second)
+ if err := os.Chtimes(f.imagePath, later, later); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// countReads returns a reader of how many times the image has been hashed
+// since this call.
+func (f *rootfsBox) countReads(t *testing.T) func() int {
+ t.Helper()
+ before := f.srv.imageDigests.hashed()
+ return func() int { return f.srv.imageDigests.hashed() - before }
+}
+
+func (f *rootfsBox) readLaunch(t *testing.T, dir string) RootfsLaunch {
+ t.Helper()
+ raw, err := os.ReadFile(filepath.Join(dir, RootfsExeLaunch))
+ if err != nil {
+ t.Fatalf("launch.json: %v", err)
+ }
+ var l RootfsLaunch
+ if err := json.Unmarshal(raw, &l); err != nil {
+ t.Fatalf("launch.json: %v: %s", err, raw)
+ }
+ return l
+}
+
+// populate writes the host slice's cgroup.events as slicePopulated reads it.
+func (f *rootfsBox) populate(t *testing.T, populated bool) {
+ t.Helper()
+ ev := filepath.Join(f.srv.CgroupRoot, parentSlices, SliceName(f.host), "cgroup.events")
+ if err := os.MkdirAll(filepath.Dir(ev), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ v := "populated 0\nfrozen 0\n"
+ if populated {
+ v = "populated 1\nfrozen 0\n"
+ }
+ if err := os.WriteFile(ev, []byte(v), 0o644); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// warm makes one successful run start, so the reload a fresh Server owes
+// is behind us, and forgets the calls it made.
+func (f *rootfsBox) warm(t *testing.T) {
+ t.Helper()
+ if status, body := f.post(t, "/start", f.startQuery(RoleRun, release.ProcessWeb, 0)); status != 200 {
+ t.Fatalf("warm start: %d %s", status, body)
+ }
+ f.ctl.mustHaveCall(t, "daemon-reload")
+ f.ctl.reset()
+}
+
+// answerView answers the rootfsView show for unit explicitly.
+func (f *rootfsBox) answerView(t *testing.T, unit string, applied bool) {
+ t.Helper()
+ f.ctl.set(rootfsViewShow+" "+unit+" "+SliceName(f.host), rootfsViewAnswer(f.host, unit, applied))
+}
+
+func (f *rootfsBox) plantJITDropIn(t *testing.T, unit string) {
+ t.Helper()
+ dir := filepath.Join(f.srv.DropInDir, unit+".d")
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(filepath.Join(dir, JITDropInName), []byte(JITDropIn), 0o644); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// answerStart answers one role's start path reads explicitly: the view
+// show for a gen or worker (applied) and the post-start show, with
+// MemoryDenyWriteExecute=no when jit.
+func (f *rootfsBox) answerStart(t *testing.T, unit string, jit bool) {
+ t.Helper()
+ if rootImageOf(unit) != "" {
+ f.answerView(t, unit, true)
+ }
+ mdwe := "yes"
+ if jit {
+ mdwe = "no"
+ }
+ f.ctl.set(rootfsPostStartShow+" "+unit, "InvocationID="+strings.Repeat("ab", 16)+"\nSlice="+SliceName(f.host)+"\nControlGroup=/carlos.slice/carlos-instances.slice/"+SliceName(f.host)+"/"+unit+"\nMemoryDenyWriteExecute="+mdwe+"\n")
+}
+
+func (f *rootfsBox) unitFor(role string) string {
+ switch role {
+ case "gen":
+ return RootfsGenUnitName(f.host, 2)
+ case "worker":
+ return WorkerUnitName(f.host, "jobs")
+ case "cmd":
+ return RootfsUnitName(RoleCmd, f.host)
+ }
+ return RootfsUnitName(RoleRun, f.host)
+}
+
+func (f *rootfsBox) keyFor(role string) string {
+ switch role {
+ case "gen":
+ return GenKey(f.host, 2)
+ case "worker":
+ return WorkerKey(f.host, "jobs")
+ }
+ return provision.InstanceID(f.host)
+}
+
+func roleOf(role string) Role {
+ switch role {
+ case "worker":
+ return RoleWorker
+ case "cmd":
+ return RoleCmd
+ }
+ return RoleRun
+}
+
+func (f *rootfsBox) startRole(t *testing.T, role string) (int, string) {
+ t.Helper()
+ switch role {
+ case "gen":
+ return f.post(t, "/start", f.startQuery(RoleRun, release.ProcessWeb, 2))
+ case "worker":
+ q := f.startQuery(RoleWorker, "jobs", 0)
+ q.Set("worker", "jobs")
+ return f.post(t, "/start", q)
+ case "cmd":
+ return f.post(t, "/cmd", f.cmdQuery("release", nil))
+ }
+ return f.post(t, "/start", f.startQuery(RoleRun, release.ProcessWeb, 0))
+}
+
+func (f *rootfsBox) stopRole(t *testing.T, role string) (int, string) {
+ t.Helper()
+ q := url.Values{"host": {f.host}, "runtime": {"rootfs"}, "role": {string(roleOf(role))}}
+ switch role {
+ case "gen":
+ q.Set("gen", "2")
+ case "worker":
+ q.Set("worker", "jobs")
+ }
+ return f.post(t, "/stop", q)
+}
+
+// The fake's assertions over its recorded calls, by leading prefix.
+
+func (f *fakeCtl) snapshot() []string {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ return append([]string(nil), f.calls...)
+}
+
+func (f *fakeCtl) mustHaveCall(t *testing.T, prefix string) {
+ t.Helper()
+ if !ctlCalled(f, prefix) {
+ t.Errorf("no systemctl call starts with %q:\n%s", prefix, f.joined())
+ }
+}
+
+func (f *fakeCtl) mustNotHaveCall(t *testing.T, prefix string) {
+ t.Helper()
+ if ctlCalled(f, prefix) {
+ t.Errorf("a systemctl call starts with %q:\n%s", prefix, f.joined())
+ }
+}
+
+func (f *fakeCtl) mustHaveCallsInOrder(t *testing.T, prefixes ...string) {
+ t.Helper()
+ f.mustHaveCallsInOrderFrom(t, 0, prefixes...)
+}
+
+// mustHaveCallsInOrderFrom finds each prefix after the one before it, over
+// calls[from:].
+func (f *fakeCtl) mustHaveCallsInOrderFrom(t *testing.T, from int, prefixes ...string) {
+ t.Helper()
+ calls := f.snapshot()
+ if from > len(calls) {
+ from = len(calls)
+ }
+ calls = calls[from:]
+ i := 0
+ for _, p := range prefixes {
+ for i < len(calls) && !strings.HasPrefix(calls[i], p) {
+ i++
+ }
+ if i == len(calls) {
+ t.Errorf("calls from %d lack %q in order %v:\n%s", from, p, prefixes, strings.Join(calls, "\n"))
+ return
+ }
+ i++
+ }
+}
+
+func (f *fakeCtl) mustNotHaveCallFrom(t *testing.T, from int, prefix string) {
+ t.Helper()
+ calls := f.snapshot()
+ if from > len(calls) {
+ from = len(calls)
+ }
+ for _, c := range calls[from:] {
+ if strings.HasPrefix(c, prefix) {
+ t.Errorf("a systemctl call after %d starts with %q:\n%s", from, prefix, strings.Join(calls, "\n"))
+ return
+ }
+ }
+}
+
+func (f *fakeCtl) callCount() int { return len(f.snapshot()) }
+
+func TestRootfsStartLaysOutRootsLaunchAndStartsTheRootfsUnit(t *testing.T) {
+ f := rootfsFixture(t)
+ status, body := f.post(t, "/start", f.startQuery(RoleRun, release.ProcessWeb, 0))
+ if status != 200 {
+ t.Fatalf("start: %d %s", status, body)
+ }
+ unit := RootfsUnitName(RoleRun, f.host)
+ f.ctl.mustHaveCall(t, "start "+unit)
+ f.ctl.mustNotHaveCall(t, "start "+UnitName(RoleRun, f.host))
+ f.ctl.mustHaveCallsInOrder(t, "start "+unit, rootfsPostStartShow+" "+unit)
+ var resp StartResponse
+ if err := json.Unmarshal([]byte(body), &resp); err != nil || len(resp.InvocationID) != 32 {
+ t.Errorf("response %s: %v", body, err)
+ }
+ dir := RootfsExeDir(f.srv.ExeRoot, provision.InstanceID(f.host), RoleRun)
+ if target, err := os.Readlink(filepath.Join(dir, RootfsExeImage)); err != nil || target != f.imagePath {
+ t.Errorf("image link = %q, %v", target, err)
+ }
+ l := f.readLaunch(t, dir)
+ if l.Schema != RootfsLaunchSchema || l.Host != f.host || l.Process != release.ProcessWeb || !reflect.DeepEqual(l.Argv, []string{"/app/serve"}) || l.Version != rootfsVer || !strings.EqualFold(l.Digest, f.digest) || l.WorkingDir != "/app" || l.Env["RAILS_ENV"] != "production" || l.JIT {
+ t.Errorf("launch = %+v", l)
+ }
+ if b, _ := os.ReadFile(filepath.Join(dir, RootfsExeResolv)); string(b) != f.resolvConf {
+ t.Errorf("resolv.conf copy = %q", b)
+ }
+ if fi, _ := os.Stat(filepath.Join(dir, RootfsExeLaunch)); fi.Mode().Perm() != 0o644 {
+ t.Errorf("launch.json mode %o", fi.Mode().Perm())
+ }
+ // A run unit is a template unit: no generation drop-in, and no jit one
+ // for a release that declares none.
+ if entries, err := os.ReadDir(filepath.Join(f.srv.DropInDir, unit+".d")); err == nil && len(entries) > 0 {
+ t.Errorf("a run start wrote drop-ins: %v", entries)
+ }
+}
+
+// C4: the command comes from the root-written staged manifest and nowhere
+// else. A process the manifest does not declare, a manifest that is not a
+// rootfs one, an image whose digest is not the manifest's, a path that is
+// not the host's own staged copy, and a row that is not a rootfs row are
+// each refused with 403 before anything is laid out or started.
+func TestRootfsStartRefusals(t *testing.T) {
+ cases := []struct {
+ name string
+ mut func(t *testing.T, f *rootfsBox)
+ query func(f *rootfsBox) url.Values
+ want string
+ }{
+ {"undeclared process", nil, func(f *rootfsBox) url.Values { return f.startQuery(RoleRun, "cron", 0) }, "cron is not a process this release declares"},
+ {"worker asked as web", nil, func(f *rootfsBox) url.Values { return f.startQuery(RoleRun, "jobs", 0) }, "run role is web"},
+ {"no process", nil, func(f *rootfsBox) url.Values { return f.startQuery(RoleRun, "", 0) }, "run role is web"},
+ {"binary manifest", func(t *testing.T, f *rootfsBox) {
+ f.restageManifest(t, func(m *release.Manifest) {
+ m.Kind = "binary"
+ m.Artifacts = map[string]release.Artifact{"linux-arm64": m.Artifacts[release.RootfsArtifactKey]}
+ m.Processes = nil
+ })
+ }, nil, "is a binary, not a rootfs image"},
+ {"digest not the manifest's", func(t *testing.T, f *rootfsBox) {
+ f.restageManifest(t, func(m *release.Manifest) {
+ a := m.Artifacts[release.RootfsArtifactKey]
+ a.SHA256 = strings.Repeat("0", 64)
+ m.Artifacts[release.RootfsArtifactKey] = a
+ })
+ }, nil, "digest"},
+ {"bytes not the digest", func(t *testing.T, f *rootfsBox) { f.rewriteImage(t, []byte(strings.Repeat("x", len(f.image)))) }, nil, "digest"},
+ {"another app's image", nil, func(f *rootfsBox) url.Values {
+ q := f.startQuery(RoleRun, release.ProcessWeb, 0)
+ q.Set("image", release.VersionPath(f.srv.OptDir, rootfsAcct, rootfsVer, "blog"))
+ return q
+ }, "is not shop.example.com's staged v1"},
+ {"a symlink in the path", func(t *testing.T, f *rootfsBox) { f.symlinkVersionDir(t) }, nil, "symlink"},
+ {"not a rootfs row", func(t *testing.T, f *rootfsBox) { f.srv.HostRuntime = func(string) (string, error) { return "", nil } }, nil, "is not a rootfs instance on this box"},
+ {"no registry view", func(t *testing.T, f *rootfsBox) { f.srv.HostRuntime = nil }, nil, "no registry view"},
+ {"missing sibling", func(t *testing.T, f *rootfsBox) {
+ if err := os.Remove(release.StagedManifestPath(f.srv.OptDir, rootfsAcct, rootfsVer, rootfsApp)); err != nil {
+ t.Fatal(err)
+ }
+ }, nil, "staged manifest"},
+ {"image without runtime", nil, func(f *rootfsBox) url.Values {
+ q := f.startQuery(RoleRun, release.ProcessWeb, 0)
+ q.Del("runtime")
+ return q
+ }, "runtime"},
+ {"runtime without image", nil, func(f *rootfsBox) url.Values {
+ q := f.startQuery(RoleRun, release.ProcessWeb, 0)
+ q.Del("image")
+ q.Del("version")
+ q.Del("digest")
+ return q
+ }, "image"},
+ {"an exe beside the image", nil, func(f *rootfsBox) url.Values {
+ q := f.startQuery(RoleRun, release.ProcessWeb, 0)
+ q.Set("exe", f.imagePath)
+ return q
+ }, "exe"},
+ {"a sidecar from an image", nil, func(f *rootfsBox) url.Values { return f.startQuery(RoleSidecar, release.ProcessWeb, 0) }, "sidecar"},
+ {"a worker without its name", nil, func(f *rootfsBox) url.Values { return f.startQuery(RoleWorker, "jobs", 0) }, "worker"},
+ {"a worker not declared", nil, func(f *rootfsBox) url.Values {
+ q := f.startQuery(RoleWorker, "cron", 0)
+ q.Set("worker", "cron")
+ return q
+ }, "cron is not a process this release declares"},
+ {"a worker named web", nil, func(f *rootfsBox) url.Values {
+ q := f.startQuery(RoleWorker, "web", 0)
+ q.Set("worker", "web")
+ return q
+ }, "worker"},
+ {"a worker whose process is another", nil, func(f *rootfsBox) url.Values {
+ q := f.startQuery(RoleWorker, "release", 0)
+ q.Set("worker", "jobs")
+ return q
+ }, "jobs"},
+ {"a bad version", nil, func(f *rootfsBox) url.Values {
+ q := f.startQuery(RoleRun, release.ProcessWeb, 0)
+ q.Set("version", "../v1")
+ return q
+ }, "version"},
+ }
+ for _, c := range cases {
+ t.Run(c.name, func(t *testing.T) {
+ f := rootfsFixture(t)
+ if c.mut != nil {
+ c.mut(t, f)
+ }
+ q := f.startQuery(RoleRun, release.ProcessWeb, 0)
+ if c.query != nil {
+ q = c.query(f)
+ }
+ status, body := f.post(t, "/start", q)
+ if status != 403 && status != 400 {
+ t.Fatalf("status %d, want 403 or 400: %s", status, body)
+ }
+ if !strings.Contains(body, c.want) {
+ t.Errorf("body %q lacks %q", body, c.want)
+ }
+ for _, call := range f.ctl.snapshot() {
+ if strings.HasPrefix(call, "start ") || strings.HasPrefix(call, "set-property") || strings.HasPrefix(call, "daemon-reload") {
+ t.Errorf("systemctl %q ran after a refusal", call)
+ }
+ }
+ for _, role := range []Role{RoleRun, RoleWorker} {
+ for _, key := range []string{provision.InstanceID(f.host), WorkerKey(f.host, "jobs"), WorkerKey(f.host, "cron")} {
+ if _, err := os.Stat(RootfsExeDir(f.srv.ExeRoot, key, role)); err == nil {
+ t.Errorf("exe layout %s/%s written after a refusal", key, role)
+ }
+ }
+ }
+ })
+ }
+}
+
+func TestRootfsStartRefusesWhileTheRootfsProbeHasNotPassed(t *testing.T) {
+ f := rootfsFixture(t)
+ f.srv.setRootfsReport(ProbeReport{Ready: false, Lines: []VerdictLine{{Check: "root-image", Verdict: "fail", Detail: "ext4"}}})
+ status, body := f.post(t, "/start", f.startQuery(RoleRun, release.ProcessWeb, 0))
+ if status != 503 || !strings.Contains(body, "this box has not passed the rootfs isolation probe on this boot (root-image)") {
+ t.Fatalf("status %d body %q", status, body)
+ }
+ f.ctl.mustNotHaveCall(t, "start ")
+ if _, err := os.Stat(RootfsExeDir(f.srv.ExeRoot, provision.InstanceID(f.host), RoleRun)); err == nil {
+ t.Error("exe layout written while the probe has not passed")
+ }
+ // The binary probe counts too: a box that is not nested-ready cannot be
+ // rootfs-ready, and the report says which probe is the reason.
+ f.srv.setRootfsReport(ProbeReport{Ready: true, Lines: []VerdictLine{{Check: "root-image", Verdict: "ok"}}})
+ f.srv.mu.Lock()
+ f.srv.report = ProbeReport{Ready: false, Lines: []VerdictLine{{Check: "pid1", Verdict: "fail", Detail: "pid=4242"}}}
+ f.srv.mu.Unlock()
+ if status, body := f.post(t, "/start", f.startQuery(RoleRun, release.ProcessWeb, 0)); status != 503 || !strings.Contains(body, "(nested)") {
+ t.Fatalf("binary probe failing: %d %q", status, body)
+ }
+ rep, ok := f.srv.ReadyRootfs()
+ if ok || len(rep.Lines) == 0 || rep.Lines[0].Check != "nested" || rep.Lines[0].Verdict != "fail" || !strings.Contains(rep.Lines[0].Detail, "pid1") {
+ t.Errorf("ReadyRootfs with the binary probe failing = %+v, %v", rep, ok)
+ }
+ if status, body := f.get(t, RootfsReadyPath, nil, nil), ""; status != 503 {
+ t.Errorf("GET %s: %d %s", RootfsReadyPath, status, body)
+ }
+}
+
+func TestRootfsDigestIsCachedByIdentityAndRehashedOnChange(t *testing.T) {
+ f := rootfsFixture(t)
+ reads := f.countReads(t)
+ if _, err := f.srv.validateStagedImage(f.host, RoleRun, f.img(release.ProcessWeb)); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := f.srv.validateStagedImage(f.host, RoleRun, f.img(release.ProcessWeb)); err != nil {
+ t.Fatal(err)
+ }
+ if n := reads(); n != 1 {
+ t.Errorf("the image was read %d times for two validations; the cache should make it one", n)
+ }
+ // A rewritten file (same inode, same size, new mtime) is hashed again
+ // and refused.
+ f.rewriteImage(t, []byte(strings.Repeat("x", len(f.image))))
+ if _, err := f.srv.validateStagedImage(f.host, RoleRun, f.img(release.ProcessWeb)); !errors.Is(err, ErrStagedImage) {
+ t.Fatalf("changed image accepted from the cache: %v", err)
+ }
+ if n := reads(); n != 2 {
+ t.Errorf("the rewritten image was not hashed again: %d reads", n)
+ }
+}
+
+// Preflight 2.11: a worker start whose view already applies starts with no
+// daemon-reload (wake latency); one whose view does not apply reloads once,
+// re-reads, and starts nothing when it still does not apply.
+func TestRootfsWorkerStartWritesItsDropInAndProvesIt(t *testing.T) {
+ f := rootfsFixture(t)
+ f.warm(t)
+ unit := WorkerUnitName(f.host, "jobs")
+ f.answerStart(t, unit, false)
+ q := f.startQuery(RoleWorker, "jobs", 0)
+ q.Set("worker", "jobs")
+ status, body := f.post(t, "/start", q)
+ if status != 200 {
+ t.Fatalf("worker start: %d %s", status, body)
+ }
+ got, err := os.ReadFile(filepath.Join(f.srv.DropInDir, unit+".d", DropInName))
+ if err != nil || string(got) != WorkerDropIn(f.host, "jobs", cgroup.TierChildren) {
+ t.Errorf("drop-in:\n%s\n%v", got, err)
+ }
+ f.ctl.mustHaveCallsInOrder(t, rootfsViewShow+" "+unit, "start "+unit)
+ f.ctl.mustNotHaveCall(t, "daemon-reload")
+ if strings.Contains(string(got), "MemoryMax=") {
+ t.Error("a worker drop-in caps memory; workers share the host slice's tier")
+ }
+ dir := RootfsExeDir(f.srv.ExeRoot, WorkerKey(f.host, "jobs"), RoleWorker)
+ if l := f.readLaunch(t, dir); l.Process != "jobs" || !reflect.DeepEqual(l.Argv, []string{"/app/jobs"}) {
+ t.Errorf("worker launch = %+v", l)
+ }
+ // The drop-in that does not apply: one reload, a second look, no start.
+ f2 := rootfsFixture(t)
+ f2.warm(t)
+ f2.answerView(t, unit, false)
+ q = f2.startQuery(RoleWorker, "jobs", 0)
+ q.Set("worker", "jobs")
+ if status, body := f2.post(t, "/start", q); status != 502 || !strings.Contains(body, "did not apply") {
+ t.Errorf("a drop-in that did not apply answered %d %s", status, body)
+ }
+ f2.ctl.mustHaveCallsInOrder(t, rootfsViewShow, "daemon-reload", rootfsViewShow)
+ f2.ctl.mustNotHaveCall(t, "start "+unit)
+ if _, err := os.Stat(filepath.Join(f2.srv.ExeRoot, WorkerKey(f2.host, "jobs"))); err == nil {
+ t.Error("the refused worker kept its exe layout")
+ }
+ if _, err := os.Stat(filepath.Join(f2.srv.DropInDir, unit+".d")); err == nil {
+ t.Error("the refused worker kept its drop-in")
+ }
+}
+
+// Task 4's carry: a gen or worker template has no RootImage= without its
+// drop-in, so the view must show the drop-in's image, not only its slice.
+func TestRootfsGenStartRefusesAViewWithoutItsRootImage(t *testing.T) {
+ f := rootfsFixture(t)
+ f.warm(t)
+ unit := RootfsGenUnitName(f.host, 2)
+ answer := strings.Replace(rootfsViewAnswer(f.host, unit, true), "RootImage="+rootImageOf(unit), "RootImage=", 1)
+ f.ctl.set(rootfsViewShow+" "+unit+" "+SliceName(f.host), answer)
+ if status, body := f.post(t, "/start", f.startQuery(RoleRun, release.ProcessWeb, 2)); status != 502 || !strings.Contains(body, "did not apply") {
+ t.Fatalf("a view with no RootImage answered %d %s", status, body)
+ }
+ f.ctl.mustHaveCall(t, "daemon-reload")
+ f.ctl.mustNotHaveCall(t, "start "+unit)
+ if _, err := os.Stat(filepath.Join(f.srv.ExeRoot, GenKey(f.host, 2))); err == nil {
+ t.Error("the refused generation kept its exe layout")
+ }
+}
+
+// Preflight 1.1: role=worker is a rootfs role only. Through Handler(), every
+// binary verb answers 400 to it and no systemctl call runs.
+func TestBinaryVerbsRefuseTheWorkerRole(t *testing.T) {
+ f := rootfsFixture(t)
+ for _, c := range []struct{ method, path string }{{"POST", "/start"}, {"POST", "/seed"}, {"POST", "/stop"}, {"POST", "/reset"}, {"POST", "/cmd"}, {"GET", "/cmd/" + f.host}} {
+ q := url.Values{"host": {f.host}, "role": {"worker"}, "worker": {"jobs"}, "memory_max": {strconv.FormatInt(cgroup.TierChildren.MemoryMax, 10)}, "invocation": {"x"}}
+ // The parser's own refusal, naming what is missing: a later guard
+ // (UnitName answering "") would refuse too, but without the reason.
+ status, body := f.do(t, c.method, c.path, q)
+ if status != 400 || !strings.Contains(body, "worker") || !strings.Contains(body, "runtime=rootfs") {
+ t.Errorf("%s %s role=worker: %d %s, want 400 naming runtime=rootfs", c.method, c.path, status, body)
+ }
+ // A worker name without the role is refused the same way.
+ q.Del("role")
+ if status, body := f.do(t, c.method, c.path, q); status != 400 || !strings.Contains(body, "worker") || !strings.Contains(body, "runtime=rootfs") {
+ t.Errorf("%s %s worker= without role: %d %s, want 400 naming runtime=rootfs", c.method, c.path, status, body)
+ }
+ }
+ if n := f.ctl.callCount(); n != 0 {
+ t.Errorf("systemctl ran: %v", f.ctl.snapshot())
+ }
+}
+
+// Preflight 1.2: sweepGenLeftovers judges a rootfs generation's exe dir by
+// the rootfs gen unit. A rootfs gen 2 whose stop failed (in the failed map
+// under RootfsGenUnitName) keeps <ExeRoot>/<GenKey>/ whole (image link,
+// launch.json, resolv.conf); the same dir with the rootfs gen unit inactive
+// and its cgroup empty is removed; a binary gen dir beside it keeps today's
+// behaviour.
+func TestSweepKeepsAStuckRootfsGenerationsExeDir(t *testing.T) {
+ plant := func(t *testing.T, f *rootfsBox) (rootfsDir, binaryDir string) {
+ t.Helper()
+ launch := RootfsLaunch{Schema: RootfsLaunchSchema, Host: f.host, Process: release.ProcessWeb, Argv: []string{"/app/serve"}, Version: rootfsVer, Digest: f.digest}
+ if err := f.srv.ensureRootfsExe(GenKey(f.host, 2), RoleRun, f.imagePath, launch); err != nil {
+ t.Fatal(err)
+ }
+ if err := f.srv.ensureGenExe(f.host, 3); err != nil {
+ t.Fatal(err)
+ }
+ return filepath.Join(f.srv.ExeRoot, GenKey(f.host, 2)), filepath.Join(f.srv.ExeRoot, GenKey(f.host, 3))
+ }
+ whole := func(t *testing.T, dir string) {
+ t.Helper()
+ for _, name := range []string{RootfsExeImage, RootfsExeLaunch, RootfsExeResolv} {
+ if _, err := os.Lstat(filepath.Join(dir, RootfsExeDirName, string(RoleRun), name)); err != nil {
+ t.Errorf("%s: %v", name, err)
+ }
+ }
+ }
+ // Stuck: the rootfs gen unit's stop failed, so its exe dir stays whole.
+ f := rootfsFixture(t)
+ rootfsDir, binaryDir := plant(t, f)
+ stuck := RootfsGenUnitName(f.host, 2)
+ f.ctl.set("list-units", stuck+" loaded active running\n")
+ f.ctl.set("list-jobs", "")
+ f.ctl.fail["stop "+stuck] = errors.New("exit status 1")
+ status, body := f.post(t, "/sweep", nil)
+ if status != 502 {
+ t.Fatalf("sweep with a stuck unit: %d %s", status, body)
+ }
+ var out SweepResponse
+ if err := json.Unmarshal([]byte(body), &out); err != nil || out.Failed[stuck] == "" {
+ t.Fatalf("sweep response %s: %v", body, err)
+ }
+ whole(t, rootfsDir)
+ if _, err := os.Stat(binaryDir); err == nil {
+ t.Error("the inactive binary generation's exe dir survived the sweep")
+ }
+ // Gone: the rootfs gen unit is inactive and its cgroup empty, so the
+ // dir goes with the binary one; a worker's leftover directory goes the
+ // same way, judged by its worker unit.
+ f = rootfsFixture(t)
+ rootfsDir, binaryDir = plant(t, f)
+ workerDir := filepath.Join(f.srv.ExeRoot, WorkerKey(f.host, "jobs"))
+ if err := os.MkdirAll(filepath.Join(workerDir, RootfsExeDirName, string(RoleWorker)), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ f.ctl.set("list-units", "")
+ f.ctl.set("list-jobs", "")
+ if status, body := f.post(t, "/sweep", nil); status != 200 {
+ t.Fatalf("sweep: %d %s", status, body)
+ }
+ f.ctl.mustHaveCall(t, "show -p ActiveState --value "+stuck)
+ f.ctl.mustHaveCall(t, "show -p ActiveState --value "+WorkerUnitName(f.host, "jobs"))
+ f.ctl.mustNotHaveCall(t, "show -p ActiveState --value "+GenUnitName(f.host, 2))
+ for _, dir := range []string{rootfsDir, binaryDir, workerDir} {
+ if _, err := os.Stat(dir); err == nil {
+ t.Errorf("%s survived the sweep of an inactive unit", dir)
+ }
+ }
+ // A worker whose stop failed keeps its directory, as a generation does.
+ f = rootfsFixture(t)
+ workerDir = filepath.Join(f.srv.ExeRoot, WorkerKey(f.host, "jobs"))
+ if err := os.MkdirAll(filepath.Join(workerDir, RootfsExeDirName, string(RoleWorker)), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ f.ctl.set("list-units", WorkerUnitName(f.host, "jobs")+" loaded active running\n")
+ f.ctl.set("list-jobs", "")
+ f.ctl.fail["stop "+WorkerUnitName(f.host, "jobs")] = errors.New("exit status 1")
+ if status, _ := f.post(t, "/sweep", nil); status != 502 {
+ t.Fatalf("sweep with a stuck worker: %d", status)
+ }
+ if _, err := os.Stat(workerDir); err != nil {
+ t.Errorf("the stuck worker's directory went: %v", err)
+ }
+ // A rootfs gen whose unit still runs (populated cgroup) keeps its dir.
+ f = rootfsFixture(t)
+ rootfsDir, _ = plant(t, f)
+ f.ctl.set("list-units", "")
+ f.ctl.set("list-jobs", "")
+ ev := filepath.Join(f.srv.CgroupRoot, parentSlices, SliceName(f.host), stuck, "cgroup.events")
+ if err := os.MkdirAll(filepath.Dir(ev), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.WriteFile(ev, []byte("populated 1\n"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if status, body := f.post(t, "/sweep", nil); status != 200 {
+ t.Fatalf("sweep: %d %s", status, body)
+ }
+ whole(t, rootfsDir)
+}
+
+// Preflight 2.14: only a path under systemd's image root is the image's.
+func TestRootfsStartMapsOnlyImagePathsToTheImageSentence(t *testing.T) {
+ f := rootfsFixture(t)
+ unit := RootfsUnitName(RoleRun, f.host)
+ f.ctl.fail["start "+unit] = errors.New("exit status 1")
+ f.ctl.set("start "+unit, "Failed to set up mount namespacing: /run/carlos-control/exe/shop.example.com/rootfs/run/resolv.conf: No such file or directory\n")
+ status, body := f.post(t, "/start", f.startQuery(RoleRun, release.ProcessWeb, 0))
+ if status != 502 || strings.Contains(body, "the image has no") || !strings.Contains(body, "/run/carlos-control/exe") {
+ t.Fatalf("a missing bind source: %d %q, want a raw 502 naming the host path", status, body)
+ }
+}
+
+func TestRootfsMountFailure(t *testing.T) {
+ for _, c := range []struct {
+ out string
+ path string
+ ok bool
+ }{
+ {"Failed to set up mount namespacing: /run/systemd/mount-rootfs/etc/resolv.conf: No such file or directory\n", "/etc/resolv.conf", true},
+ {"Job failed. ... /run/systemd/mount-rootfs/opt/carlos: No such file or directory", "/opt/carlos", true},
+ {"Failed to set up mount namespacing: /run/carlos-control/exe/x/rootfs/run/resolv.conf: No such file or directory", "", false},
+ {"Failed to set up mount namespacing: /run/systemd/mount-rootfs/data: Permission denied", "", false},
+ {"", "", false},
+ {"/run/systemd/mount-rootfs/: No such file or directory", "", false},
+ } {
+ if path, ok := rootfsMountFailure(c.out); path != c.path || ok != c.ok {
+ t.Errorf("rootfsMountFailure(%q) = %q, %v; want %q, %v", c.out, path, ok, c.path, c.ok)
+ }
+ }
+}
+
+// C5: a unit systemd placed outside the host slice is ended, not served.
+func TestRootfsStartStopsAUnitOutsideTheHostSlice(t *testing.T) {
+ f := rootfsFixture(t)
+ unit := RootfsUnitName(RoleRun, f.host)
+ f.ctl.set(rootfsPostStartShow, "InvocationID="+strings.Repeat("ab", 16)+"\nSlice=system.slice\nControlGroup=/system.slice/"+unit+"\nMemoryDenyWriteExecute=yes\n")
+ status, body := f.post(t, "/start", f.startQuery(RoleRun, release.ProcessWeb, 0))
+ if status != 502 || !strings.Contains(body, "slice") {
+ t.Fatalf("status %d body %q", status, body)
+ }
+ f.ctl.mustHaveCallsInOrder(t, "start "+unit, rootfsPostStartShow+" "+unit, "stop "+unit)
+ // The same for a worker, whose files go with it.
+ f = rootfsFixture(t)
+ worker := WorkerUnitName(f.host, "jobs")
+ f.answerView(t, worker, true)
+ f.ctl.set(rootfsPostStartShow+" "+worker, "InvocationID="+strings.Repeat("ab", 16)+"\nSlice="+SliceName(f.host)+"\nControlGroup=/carlos.slice/carlos-instances.slice/"+SliceName(f.host)+"/../"+worker+"\nMemoryDenyWriteExecute=yes\n")
+ q := f.startQuery(RoleWorker, "jobs", 0)
+ q.Set("worker", "jobs")
+ if status, body := f.post(t, "/start", q); status != 502 || !strings.Contains(body, "slice") {
+ t.Fatalf("worker outside its slice: %d %q", status, body)
+ }
+ f.ctl.mustHaveCall(t, "stop "+worker)
+ if _, err := os.Stat(filepath.Join(f.srv.ExeRoot, WorkerKey(f.host, "jobs"))); err == nil {
+ t.Error("the stopped worker kept its exe layout")
+ }
+}
+
+func TestRootfsCmdTakesAProcessOrAnArgvNeverBoth(t *testing.T) {
+ f := rootfsFixture(t)
+ // A declared process (the release process, Plan 3's caller).
+ status, body := f.post(t, "/cmd", f.cmdQuery("release", nil))
+ if status != 200 {
+ t.Fatalf("cmd release: %d %s", status, body)
+ }
+ var resp StartResponse
+ if err := json.Unmarshal([]byte(body), &resp); err != nil || len(resp.InvocationID) != 32 {
+ t.Errorf("response %s: %v", body, err)
+ }
+ l := f.readLaunch(t, RootfsExeDir(f.srv.ExeRoot, provision.InstanceID(f.host), RoleCmd))
+ if l.Process != "release" || !reflect.DeepEqual(l.Argv, []string{"/app/migrate"}) || l.WorkingDir != "/app" || l.Env["PATH"] != "/app/bin:/usr/bin" {
+ t.Errorf("release launch = %+v", l)
+ }
+ f.ctl.mustHaveCallsInOrder(t, "start --no-block "+RootfsUnitName(RoleCmd, f.host), rootfsPostStartShow+" "+RootfsUnitName(RoleCmd, f.host))
+ f.ctl.mustNotHaveCall(t, "start --no-block "+UnitName(RoleCmd, f.host))
+ // An operator argv (carlos run). Astra R1 finding 7: it keeps its argv
+ // and inherits web's env and working_dir, so a relative command resolves
+ // where the app's own process would run it (/app/bin/rails, Task 6's
+ // rule) and a bare name resolves along the image's own PATH
+ // (/usr/local/bundle/bin), never from / or the default path.
+ f = rootfsFixture(t)
+ if status, body := f.post(t, "/cmd", f.cmdQuery("", []string{"bin/rails", "db:migrate:status"})); status != 200 {
+ t.Fatalf("cmd argv: %d %s", status, body)
+ }
+ l = f.readLaunch(t, RootfsExeDir(f.srv.ExeRoot, provision.InstanceID(f.host), RoleCmd))
+ if l.Process != "cmd" || !reflect.DeepEqual(l.Argv, []string{"bin/rails", "db:migrate:status"}) {
+ t.Errorf("argv launch = %+v", l)
+ }
+ if l.WorkingDir != "/app" || l.Env["PATH"] != "/app/bin:/usr/local/bundle/bin:/usr/bin" || l.Env["RAILS_ENV"] != "production" {
+ t.Errorf("an explicit argv did not inherit web's working_dir and env: %+v", l)
+ }
+ // A worker's one-off run through cmd.
+ f = rootfsFixture(t)
+ if status, body := f.post(t, "/cmd", f.cmdQuery("jobs", nil)); status != 200 {
+ t.Fatalf("cmd jobs: %d %s", status, body)
+ }
+ for _, bad := range []struct {
+ process string
+ argv []string
+ want string
+ }{{"", nil, "process"}, {"release", []string{"x"}, "process"}, {release.ProcessWeb, nil, "web"}, {"", []string{""}, "argv"}, {"cron", nil, "cron is not a process this release declares"}} {
+ f := rootfsFixture(t)
+ status, body := f.post(t, "/cmd", f.cmdQuery(bad.process, bad.argv))
+ if status != 400 && status != 403 {
+ t.Errorf("cmd process=%q argv=%v answered %d", bad.process, bad.argv, status)
+ }
+ if !strings.Contains(body, bad.want) {
+ t.Errorf("cmd process=%q argv=%v: %q lacks %q", bad.process, bad.argv, body, bad.want)
+ }
+ f.ctl.mustNotHaveCall(t, "start")
+ }
+ // A body over the bound is refused before anything runs.
+ f = rootfsFixture(t)
+ q := f.cmdQuery("", nil)
+ q.Set("_argv", `{"argv":["`+strings.Repeat("a", 65<<10)+`"]}`)
+ if status, _ := f.post(t, "/cmd", q); status != 400 && status != 413 {
+ t.Errorf("an oversized argv body answered %d", status)
+ }
+ f.ctl.mustNotHaveCall(t, "start")
+}
+
+func TestRootfsStartMapsAMissingMountPointToItsPath(t *testing.T) {
+ f := rootfsFixture(t)
+ unit := RootfsUnitName(RoleRun, f.host)
+ f.ctl.fail["start "+unit] = errors.New("exit status 1")
+ f.ctl.set("start "+unit, "Job for carlos-instance-rootfs@shop.example.com.service failed. See \"systemctl status\" ... Failed to set up mount namespacing: /run/systemd/mount-rootfs/etc/resolv.conf: No such file or directory\n")
+ status, body := f.post(t, "/start", f.startQuery(RoleRun, release.ProcessWeb, 0))
+ if status != 502 || !strings.Contains(body, "the image has no /etc/resolv.conf, which the instance needs; rebuild it with carlos ship") {
+ t.Fatalf("status %d body %q", status, body)
+ }
+}
+
+func TestRootfsSweepCoversTheRootfsTemplatesAndTheirFiles(t *testing.T) {
+ f := rootfsFixture(t)
+ w := WorkerUnitName(f.host, "jobs")
+ g := RootfsGenUnitName(f.host, 2)
+ c := RootfsUnitName(RoleCmd, f.host)
+ f.ctl.set("list-units", RootfsUnitName(RoleRun, f.host)+" loaded active running\n"+w+" loaded active running\n"+g+" loaded failed failed\n"+c+" loaded active exited\n")
+ f.ctl.set("list-jobs", "")
+ f.ctl.set("show -p ActiveState --value", "inactive\n")
+ // Worker, generation and cmd layouts, planted as a start would, each
+ // with a jit drop-in.
+ for _, key := range []string{WorkerKey(f.host, "jobs"), GenKey(f.host, 2)} {
+ if err := os.MkdirAll(filepath.Join(f.srv.ExeRoot, key, RootfsExeDirName), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if err := os.MkdirAll(RootfsExeDir(f.srv.ExeRoot, provision.InstanceID(f.host), RoleCmd), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ for _, unit := range []string{RootfsUnitName(RoleRun, f.host), w, g, c} {
+ f.plantJITDropIn(t, unit)
+ }
+ status, body := f.post(t, "/sweep", nil)
+ if status != 200 {
+ t.Fatalf("sweep: %d %s", status, body)
+ }
+ for _, unit := range []string{RootfsUnitName(RoleRun, f.host), w, g, c} {
+ f.ctl.mustHaveCall(t, "stop "+unit)
+ if _, err := os.Stat(filepath.Join(f.srv.DropInDir, unit+".d", JITDropInName)); err == nil {
+ t.Errorf("%s kept its jit drop-in through the sweep", unit)
+ }
+ }
+ for _, key := range []string{WorkerKey(f.host, "jobs"), GenKey(f.host, 2)} {
+ if _, err := os.Stat(filepath.Join(f.srv.ExeRoot, key)); err == nil {
+ t.Errorf("exe directory %s survived the sweep", key)
+ }
+ }
+ if _, err := os.Stat(RootfsExeDir(f.srv.ExeRoot, provision.InstanceID(f.host), RoleCmd)); err == nil {
+ t.Error("the rootfs cmd layout survived the sweep")
+ }
+ // Review Focus 4: the list-units glob must name the rootfs prefixes.
+ listed := false
+ for _, call := range f.ctl.snapshot() {
+ if strings.HasPrefix(call, "list-units") {
+ listed = true
+ for _, p := range []string{UnitPrefixRootfs, UnitPrefixRootfsGen, UnitPrefixRootfsWorker, UnitPrefixRootfsCmd} {
+ if !strings.Contains(call, p+"*.service") {
+ t.Errorf("sweep does not list %s", p)
+ }
+ }
+ }
+ }
+ if !listed {
+ t.Error("no list-units call")
+ }
+ // Removing a unit's jit drop-in leaves the next rootfs start a reload.
+ if !f.srv.jitReloadOwed() {
+ t.Error("the sweep removed jit drop-ins without marking a reload owed")
+ }
+}
+
+// Astra R1 finding 6: a termination verb carries runtime=rootfs and no
+// image, and the parser takes it; an image on a termination verb is a 400.
+func TestRootfsStopPicksTheRootfsUnit(t *testing.T) {
+ f := rootfsFixture(t)
+ for _, c := range []struct {
+ role Role
+ worker string
+ gen string
+ unit string
+ }{{RoleRun, "", "", RootfsUnitName(RoleRun, f.host)}, {RoleWorker, "jobs", "", WorkerUnitName(f.host, "jobs")}, {RoleCmd, "", "", RootfsUnitName(RoleCmd, f.host)}, {RoleRun, "", "3", RootfsGenUnitName(f.host, 3)}} {
+ q := url.Values{"host": {f.host}, "role": {string(c.role)}, "runtime": {"rootfs"}}
+ if c.worker != "" {
+ q.Set("worker", c.worker)
+ }
+ if c.gen != "" {
+ q.Set("gen", c.gen)
+ }
+ f.plantJITDropIn(t, c.unit)
+ for _, verb := range []string{"/stop", "/reset"} {
+ if status, body := f.post(t, verb, q); status != 204 {
+ t.Errorf("%s %s: %d %s", verb, c.unit, status, body)
+ }
+ }
+ f.ctl.mustHaveCall(t, "stop "+c.unit)
+ f.ctl.mustHaveCall(t, "reset-failed "+c.unit)
+ if c.role != RoleWorker {
+ f.ctl.mustNotHaveCall(t, "stop "+UnitName(c.role, f.host))
+ }
+ if _, err := os.Stat(filepath.Join(f.srv.DropInDir, c.unit+".d", JITDropInName)); err == nil {
+ t.Errorf("the stop of %s left its jit drop-in", c.unit)
+ }
+ for _, field := range []string{"image", "version", "digest", "process"} {
+ with := cloneValues(q)
+ with.Set(field, "x")
+ if status, body := f.post(t, "/stop", with); status != 400 || !strings.Contains(body, field) {
+ t.Errorf("stop with %s: %d %s, want 400", field, status, body)
+ }
+ }
+ }
+ // A stopped rootfs cmd unit loses its layout; a stopped worker its key
+ // directory.
+ launch := RootfsLaunch{Schema: RootfsLaunchSchema, Host: f.host, Process: "release", Argv: []string{"/app/migrate"}, Version: rootfsVer, Digest: f.digest}
+ if err := f.srv.ensureRootfsExe(provision.InstanceID(f.host), RoleCmd, f.imagePath, launch); err != nil {
+ t.Fatal(err)
+ }
+ if err := f.srv.ensureRootfsExe(WorkerKey(f.host, "jobs"), RoleWorker, f.imagePath, launch); err != nil {
+ t.Fatal(err)
+ }
+ f.post(t, "/stop", url.Values{"host": {f.host}, "role": {"cmd"}, "runtime": {"rootfs"}})
+ if _, err := os.Stat(RootfsExeDir(f.srv.ExeRoot, provision.InstanceID(f.host), RoleCmd)); err == nil {
+ t.Error("rootfs/cmd layout survived the stop")
+ }
+ f.post(t, "/stop", url.Values{"host": {f.host}, "role": {"worker"}, "worker": {"jobs"}, "runtime": {"rootfs"}})
+ if _, err := os.Stat(filepath.Join(f.srv.ExeRoot, WorkerKey(f.host, "jobs"))); err == nil {
+ t.Error("the worker's exe directory survived the stop")
+ }
+ // The run unit's standing layout stays, as the binary host's does.
+ f.post(t, "/stop", url.Values{"host": {f.host}, "role": {"run"}, "runtime": {"rootfs"}})
+ // A binary stop still picks the binary unit, and a bad runtime is a 400.
+ if status, _ := f.post(t, "/stop", url.Values{"host": {f.host}, "role": {"run"}}); status != 204 {
+ t.Errorf("binary stop: %d", status)
+ }
+ f.ctl.mustHaveCall(t, "stop "+UnitName(RoleRun, f.host))
+ if status, body := f.post(t, "/stop", url.Values{"host": {f.host}, "role": {"run"}, "runtime": {"oci"}}); status != 400 || !strings.Contains(body, "runtime") {
+ t.Errorf("runtime=oci: %d %s", status, body)
+ }
+ if status, body := f.post(t, "/sweep", url.Values{"runtime": {"rootfs"}}); status != 200 {
+ t.Errorf("sweep ignores a runtime: %d %s", status, body)
+ }
+}
+
+// Astra R1 finding 1, control side: with the ROOTFS cmd unit running under
+// invocation X and the binary cmd unit inactive under another, the status
+// poll with runtime=rootfs reports running, the poll without it reports
+// gone (it is the binary unit's), and the stop with runtime=rootfs ends the
+// rootfs unit. All through the real handler and the real parser.
+func TestRootfsCmdStatusAndStopFollowTheRootfsUnit(t *testing.T) {
+ f := rootfsFixture(t)
+ x, y := strings.Repeat("ab", 16), strings.Repeat("cd", 16)
+ fields := "show -p InvocationID,ActiveState,SubState,Result,ExecMainStatus,ExecMainCode "
+ f.ctl.set(fields+RootfsUnitName(RoleCmd, f.host), "InvocationID="+x+"\nActiveState=active\nSubState=running\nResult=success\nExecMainStatus=0\nExecMainCode=0\n")
+ f.ctl.set(fields+UnitName(RoleCmd, f.host), "InvocationID="+y+"\nActiveState=inactive\nSubState=dead\nResult=success\nExecMainStatus=0\nExecMainCode=exited\n")
+ f.srv.cmdWaitBound = 50 * time.Millisecond
+ var st CmdStatus
+ status := f.get(t, "/cmd/"+f.host, url.Values{"invocation": {x}, "runtime": {"rootfs"}}, &st)
+ if status != 200 || st.State != CmdRunning {
+ t.Fatalf("rootfs poll: %d %+v, want running", status, st)
+ }
+ status = f.get(t, "/cmd/"+f.host, url.Values{"invocation": {x}}, &st)
+ if status != 200 || st.State != CmdGone {
+ t.Fatalf("binary poll of a rootfs invocation: %d %+v, want gone (it is another unit)", status, st)
+ }
+ if status := f.get(t, "/cmd/"+f.host, url.Values{"invocation": {x}, "runtime": {"rootfs"}, "image": {f.imagePath}}, &st); status != 400 {
+ t.Errorf("a status poll with an image answered %d", status)
+ }
+ if status, body := f.post(t, "/stop", url.Values{"host": {f.host}, "role": {"cmd"}, "runtime": {"rootfs"}}); status != 204 {
+ t.Fatalf("stop: %d %s", status, body)
+ }
+ f.ctl.mustHaveCall(t, "stop "+RootfsUnitName(RoleCmd, f.host))
+ f.ctl.mustNotHaveCall(t, "stop "+UnitName(RoleCmd, f.host))
+}
+
+// Astra R1 finding 5: a generation, worker or cmd start never lowers a
+// populated host slice (the swap may have raised it for the overlap); a
+// gen-0 run start always sets it.
+func TestRootfsGenStartLeavesAPopulatedSliceAlone(t *testing.T) {
+ setsTier := func(calls []string) bool {
+ for _, c := range calls {
+ if strings.HasPrefix(c, "set-property") && strings.Contains(c, "MemoryMax=") {
+ return true
+ }
+ }
+ return false
+ }
+ // Populated: the generation start leaves the limit alone.
+ f := rootfsFixture(t)
+ f.populate(t, true)
+ if status, body := f.post(t, "/start", f.startQuery(RoleRun, release.ProcessWeb, 2)); status != 200 {
+ t.Fatalf("gen start: %d %s", status, body)
+ }
+ if setsTier(f.ctl.snapshot()) {
+ t.Error("a generation start lowered a populated slice's limit (the swap's raised parent)")
+ }
+ raw, err := os.ReadFile(filepath.Join(f.srv.DropInDir, RootfsGenUnitName(f.host, 2)+".d", DropInName))
+ if err != nil || string(raw) != RootfsGenDropIn(f.host, 2, cgroup.TierChildren) {
+ t.Errorf("gen drop-in = %q, %v", raw, err)
+ }
+ // Cold: the generation start sets it, as startGen does.
+ f = rootfsFixture(t)
+ f.populate(t, false)
+ if status, _ := f.post(t, "/start", f.startQuery(RoleRun, release.ProcessWeb, 2)); status != 200 || !setsTier(f.ctl.snapshot()) {
+ t.Error("a generation start on a cold slice did not set the tier")
+ }
+ // Cold but the slice already carries the tier: no set-property.
+ f = rootfsFixture(t)
+ f.populate(t, false)
+ unit := RootfsGenUnitName(f.host, 2)
+ sh := cgroup.SharesOf(cgroup.TierChildren)
+ max := strconv.FormatInt(cgroup.TierChildren.MemoryMax, 10)
+ withTier := strings.SplitN(rootfsViewAnswer(f.host, unit, true), "\n\n", 2)[0] + "\n\nSlice=carlos-instances.slice\nNeedDaemonReload=no\nMemoryMax=" + max + "\nMemoryHigh=" + strconv.FormatInt(cgroup.HighOf(cgroup.TierChildren.MemoryMax), 10) + "\nMemorySwapMax=" + max + "\nCPUWeight=" + strconv.Itoa(sh.CPUWeight) + "\nCPUQuotaPerSecUSec=" + cpuQuotaShown(sh.CPUQuotaPercent) + "\nIOWeight=" + strconv.Itoa(sh.IOWeight) + "\n"
+ f.ctl.set(rootfsViewShow+" "+unit+" "+SliceName(f.host), withTier)
+ if status, _ := f.post(t, "/start", f.startQuery(RoleRun, release.ProcessWeb, 2)); status != 200 || setsTier(f.ctl.snapshot()) {
+ t.Error("a generation start on a slice already at the tier set it again")
+ }
+ // Populated: a worker start and a cmd start leave it alone too.
+ f = rootfsFixture(t)
+ f.populate(t, true)
+ q := f.startQuery(RoleWorker, "jobs", 0)
+ q.Set("worker", "jobs")
+ if status, body := f.post(t, "/start", q); status != 200 || setsTier(f.ctl.snapshot()) {
+ t.Errorf("a worker start on a populated slice: %d %s, set-property %v", status, body, setsTier(f.ctl.snapshot()))
+ }
+ f = rootfsFixture(t)
+ f.populate(t, true)
+ if status, body := f.post(t, "/cmd", f.cmdQuery("release", nil)); status != 200 || setsTier(f.ctl.snapshot()) {
+ t.Errorf("a cmd start on a populated slice: %d %s, set-property %v", status, body, setsTier(f.ctl.snapshot()))
+ }
+ // Populated, gen 0: a run start sets it (a cold instance's own start).
+ f = rootfsFixture(t)
+ f.populate(t, true)
+ if status, _ := f.post(t, "/start", f.startQuery(RoleRun, release.ProcessWeb, 0)); status != 200 || !setsTier(f.ctl.snapshot()) {
+ t.Error("a gen-0 run start did not set the tier")
+ }
+}
+
+// R11, C10: a release that declares jit starts without
+// MemoryDenyWriteExecute through root's per-start drop-in, in every role;
+// one that does not keeps the line, and a stale jit drop-in is removed
+// before its start. Each case first makes one successful start (f.warm), so
+// the fresh Server's owed reload is spent and the daemon-reload asserted
+// below is the one the changed jit file causes (preflight 2.13).
+func TestRootfsJITReleaseStartsWithoutMDWEAndANonJITWithIt(t *testing.T) {
+ for _, role := range []string{"run", "gen", "worker", "cmd"} {
+ for _, jit := range []bool{true, false} {
+ t.Run(fmt.Sprintf("%s/jit=%v", role, jit), func(t *testing.T) {
+ f := rootfsFixture(t)
+ f.warm(t)
+ f.restageManifest(t, func(m *release.Manifest) { m.JIT = jit })
+ unit := f.unitFor(role)
+ // A stale drop-in from an earlier jit release of this app.
+ if !jit {
+ f.plantJITDropIn(t, unit)
+ }
+ f.answerStart(t, unit, jit)
+ if status, body := f.startRole(t, role); status != 200 {
+ t.Fatalf("start: %d %s", status, body)
+ }
+ _, err := os.Stat(filepath.Join(f.srv.DropInDir, unit+".d", JITDropInName))
+ if jit && err != nil {
+ t.Errorf("jit release: no %s for %s", JITDropInName, unit)
+ }
+ if !jit && err == nil {
+ t.Errorf("non-jit release: %s left for %s", JITDropInName, unit)
+ }
+ if jit {
+ if got, _ := os.ReadFile(filepath.Join(f.srv.DropInDir, unit+".d", JITDropInName)); string(got) != JITDropIn {
+ t.Errorf("jit drop-in:\n%s", got)
+ }
+ }
+ f.ctl.mustHaveCallsInOrder(t, "daemon-reload", "start")
+ l := f.readLaunch(t, RootfsExeDir(f.srv.ExeRoot, f.keyFor(role), roleOf(role)))
+ if l.JIT != jit {
+ t.Errorf("launch.json jit = %v", l.JIT)
+ }
+ // A second start that changes no file does not reload (wake
+ // latency). A cmd or worker unit must be stopped first (a
+ // loaded one is refused), and a stop of a jit unit deletes
+ // its jit drop-in, which the next start owes a reload for.
+ mark := f.ctl.callCount()
+ if role == "cmd" || role == "worker" {
+ if status, body := f.stopRole(t, role); status != 204 {
+ t.Fatalf("stop: %d %s", status, body)
+ }
+ }
+ if status, body := f.startRole(t, role); status != 200 {
+ t.Fatalf("second start: %d %s", status, body)
+ }
+ if jit && (role == "cmd" || role == "worker") {
+ f.ctl.mustHaveCallsInOrderFrom(t, mark, "daemon-reload", "start")
+ } else {
+ f.ctl.mustNotHaveCallFrom(t, mark, "daemon-reload")
+ }
+ })
+ }
+ }
+}
+
+// C10: the effective property is checked after every start; a unit whose
+// MemoryDenyWriteExecute does not match the release's jit is stopped and
+// the start answered 502, in both directions.
+func TestRootfsJITMismatchStopsTheUnit(t *testing.T) {
+ for _, c := range []struct {
+ jit bool
+ effective string
+ }{{true, "yes"}, {false, "no"}} {
+ f := rootfsFixture(t)
+ f.restageManifest(t, func(m *release.Manifest) { m.JIT = c.jit })
+ unit := RootfsUnitName(RoleRun, f.host)
+ f.ctl.set(rootfsPostStartShow, "InvocationID="+strings.Repeat("ab", 16)+"\nSlice="+SliceName(f.host)+"\nControlGroup=/carlos.slice/carlos-instances.slice/"+SliceName(f.host)+"/"+unit+"\nMemoryDenyWriteExecute="+c.effective+"\n")
+ status, body := f.post(t, "/start", f.startQuery(RoleRun, release.ProcessWeb, 0))
+ if status != 502 || !strings.Contains(body, "MemoryDenyWriteExecute") {
+ t.Fatalf("jit=%v effective=%s: %d %s", c.jit, c.effective, status, body)
+ }
+ f.ctl.mustHaveCall(t, "stop "+unit)
+ if _, err := os.Stat(filepath.Join(f.srv.DropInDir, unit+".d", JITDropInName)); err == nil {
+ t.Error("the jit drop-in survived a stopped start")
+ }
+ // The loaded configuration was stale, so the retry reloads.
+ if !f.srv.jitReloadOwed() {
+ t.Error("a mismatch did not leave a reload owed")
+ }
+ }
+}
+
+// Spec §5.11 "who decides": jit comes from the root-staged manifest and
+// nowhere else. A jit field in the query changes nothing.
+func TestRootfsJITComesOnlyFromTheStagedManifest(t *testing.T) {
+ f := rootfsFixture(t) // manifest: jit false
+ unit := RootfsUnitName(RoleRun, f.host)
+ f.answerStart(t, unit, false)
+ q := f.startQuery(RoleRun, release.ProcessWeb, 0)
+ q.Set("jit", "1")
+ if status, body := f.post(t, "/start", q); status != 200 {
+ t.Fatalf("start: %d %s", status, body)
+ }
+ if _, err := os.Stat(filepath.Join(f.srv.DropInDir, unit+".d", JITDropInName)); err == nil {
+ t.Error("a query field turned jit on")
+ }
+ if l := f.readLaunch(t, RootfsExeDir(f.srv.ExeRoot, provision.InstanceID(f.host), RoleRun)); l.JIT {
+ t.Error("launch.json says jit for a manifest that does not")
+ }
+}
+
+// Astra R2 finding 1: cleanup deletes the jit drop-in without a reload, so
+// the next non-jit start of the same unit name finds nothing to remove.
+// systemd keeps the unit loaded with MemoryDenyWriteExecute=no until a
+// reload (systemd.unit(5)); without one the post-start check would stop a
+// healthy non-jit release. f.loadedJIT models exactly that: the effective
+// property changes only on daemon-reload.
+func TestRootfsNonJITStartAfterJITCleanupReloads(t *testing.T) {
+ for _, role := range []string{"run", "worker", "cmd"} {
+ t.Run(role, func(t *testing.T) {
+ f := rootfsFixture(t)
+ f.loadedJIT(t)
+ unit := f.unitFor(role)
+ f.restageManifest(t, func(m *release.Manifest) { m.JIT = true })
+ if status, body := f.startRole(t, role); status != 200 {
+ t.Fatalf("jit start: %d %s", status, body)
+ }
+ if status, body := f.stopRole(t, role); status != 204 {
+ t.Fatalf("stop: %d %s", status, body)
+ }
+ if _, err := os.Stat(filepath.Join(f.srv.DropInDir, unit+".d", JITDropInName)); err == nil {
+ t.Fatal("the stop left the jit drop-in")
+ }
+ mark := f.ctl.callCount()
+ f.restageManifest(t, func(m *release.Manifest) { m.JIT = false })
+ if status, body := f.startRole(t, role); status != 200 {
+ t.Fatalf("non-jit start after cleanup: %d %s (it started on the cached jit setting)", status, body)
+ }
+ f.ctl.mustHaveCallsInOrderFrom(t, mark, "daemon-reload", "start")
+ })
+ }
+}
+
+// The owed reload is cleared only by a reload that succeeds. After a jit
+// start and its stop: daemon-reload fails once, so the non-jit start answers
+// 502 and no "start" follows; the next start reloads and answers 200. A new
+// Server over the same DropInDir and the same loaded model (a restarted
+// control service) reloads before its first rootfs start although no file
+// changed.
+func TestRootfsJITReloadStaysOwedUntilItSucceeds(t *testing.T) {
+ f := rootfsFixture(t)
+ f.loadedJIT(t)
+ unit := RootfsUnitName(RoleRun, f.host)
+ f.restageManifest(t, func(m *release.Manifest) { m.JIT = true })
+ if status, body := f.startRole(t, "run"); status != 200 {
+ t.Fatalf("jit start: %d %s", status, body)
+ }
+ if status, body := f.stopRole(t, "run"); status != 204 {
+ t.Fatalf("stop: %d %s", status, body)
+ }
+ f.restageManifest(t, func(m *release.Manifest) { m.JIT = false })
+ f.ctl.mu.Lock()
+ f.ctl.fail["daemon-reload"] = errors.New("exit status 1")
+ f.ctl.mu.Unlock()
+ mark := f.ctl.callCount()
+ if status, body := f.startRole(t, "run"); status != 502 || !strings.Contains(body, "daemon-reload") {
+ t.Fatalf("start with a failing reload: %d %s", status, body)
+ }
+ f.ctl.mustHaveCallsInOrderFrom(t, mark, "daemon-reload")
+ f.ctl.mustNotHaveCallFrom(t, mark, "start "+unit)
+ if !f.srv.jitReloadOwed() {
+ t.Fatal("a failed reload cleared the owed state")
+ }
+ f.ctl.mu.Lock()
+ delete(f.ctl.fail, "daemon-reload")
+ f.ctl.mu.Unlock()
+ mark = f.ctl.callCount()
+ if status, body := f.startRole(t, "run"); status != 200 {
+ t.Fatalf("start after the reload recovers: %d %s", status, body)
+ }
+ f.ctl.mustHaveCallsInOrderFrom(t, mark, "daemon-reload", "start "+unit)
+ if f.srv.jitReloadOwed() {
+ t.Error("a successful reload left the owed state")
+ }
+ // The next start changes nothing and reloads nothing.
+ if status, body := f.stopRole(t, "run"); status != 204 {
+ t.Fatalf("stop: %d %s", status, body)
+ }
+ mark = f.ctl.callCount()
+ if status, body := f.startRole(t, "run"); status != 200 {
+ t.Fatalf("ordinary start: %d %s", status, body)
+ }
+ f.ctl.mustNotHaveCallFrom(t, mark, "daemon-reload")
+ // A restarted control service cannot know what systemd holds loaded, so
+ // it reloads once before its first rootfs start.
+ if status, body := f.stopRole(t, "run"); status != 204 {
+ t.Fatalf("stop: %d %s", status, body)
+ }
+ f2 := f.restart(t)
+ if !f2.srv.jitReloadOwed() {
+ t.Fatal("a fresh control service owes no reload")
+ }
+ mark = f2.ctl.callCount()
+ if status, body := f2.startRole(t, "run"); status != 200 {
+ t.Fatalf("first start after a restart: %d %s", status, body)
+ }
+ f2.ctl.mustHaveCallsInOrderFrom(t, mark, "daemon-reload", "start "+unit)
+}
+
+// A removal that lands while a reload runs stays owed: the reload began
+// before it, so systemd may still hold the removed drop-in's unit loaded.
+func TestRootfsJITRemovalDuringAReloadStaysOwed(t *testing.T) {
+ f := rootfsFixture(t)
+ f.ctl.mu.Lock()
+ f.ctl.before = map[string]func(){"daemon-reload": func() { f.srv.removeJITDropIn(WorkerUnitName(f.host, "jobs")) }}
+ f.ctl.mu.Unlock()
+ f.plantJITDropIn(t, WorkerUnitName(f.host, "jobs"))
+ if err := f.srv.jitDaemonReload(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ if !f.srv.jitReloadOwed() {
+ t.Error("a removal during the reload was counted as reloaded")
+ }
+ if err := f.srv.jitDaemonReload(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ if f.srv.jitReloadOwed() {
+ t.Error("the second reload did not clear the owed state")
+ }
+}
+
+func TestRootfsVerbsAnswer503UnderClaimControl(t *testing.T) {
+ f := rootfsFixture(t)
+ f.srv.ClaimStop = http.NotFoundHandler()
+ for _, path := range []string{"/start", "/stop", "/reset", "/cmd"} {
+ if status, _ := f.post(t, path, f.startQuery(RoleRun, release.ProcessWeb, 0)); status != 503 {
+ t.Errorf("%s under claim control answered %d", path, status)
+ }
+ }
+ if status := f.get(t, "/cmd/"+f.host, url.Values{"invocation": {"x"}, "runtime": {"rootfs"}}, nil); status != 503 {
+ t.Errorf("GET /cmd under claim control answered %d", status)
+ }
+ // /ready/rootfs is not claim-aware, like /ready.
+ if status := f.get(t, RootfsReadyPath, nil, nil); status != 204 {
+ t.Errorf("GET %s under claim control answered %d", RootfsReadyPath, status)
+ }
+ f.ctl.mustNotHaveCall(t, "start ")
+}
+
+// Astra R1 finding 6: every client verb reaches the REAL parser. The
+// client is wired to f.srv.Handler() over httptest, so a verb whose query
+// the parser refuses fails here, not on a box. Preflight 1.8: a 403 from
+// root (here, a digest the manifest does not name) comes back from
+// StartRootfs, StartRootfsWorker and CmdRootfs as an error that errors.Is
+// ErrStagedImage.
+func TestClientRootfsVerbsReachTheRealParser(t *testing.T) {
+ f := rootfsFixture(t)
+ c := f.client(t)
+ ctx := context.Background()
+ inv, err := c.StartRootfs(ctx, f.host, 0, cgroup.TierChildren.MemoryMax, f.img(release.ProcessWeb))
+ if err != nil || len(inv) != 32 {
+ t.Fatalf("StartRootfs: %q %v", inv, err)
+ }
+ f.ctl.mustHaveCall(t, "start "+RootfsUnitName(RoleRun, f.host))
+ if inv, err := c.StartRootfs(ctx, f.host, 3, cgroup.TierChildren.MemoryMax, f.img(release.ProcessWeb)); err != nil || len(inv) != 32 {
+ t.Fatalf("StartRootfs gen 3: %q %v", inv, err)
+ }
+ f.ctl.mustHaveCall(t, "start "+RootfsGenUnitName(f.host, 3))
+ if inv, err := c.StartRootfsWorker(ctx, f.host, "jobs", cgroup.TierChildren.MemoryMax, f.img("jobs")); err != nil || len(inv) != 32 {
+ t.Fatalf("StartRootfsWorker: %q %v", inv, err)
+ }
+ f.ctl.mustHaveCall(t, "start "+WorkerUnitName(f.host, "jobs"))
+ for _, c2 := range []struct {
+ role Role
+ gen uint64
+ worker string
+ unit string
+ }{{RoleRun, 0, "", RootfsUnitName(RoleRun, f.host)}, {RoleRun, 3, "", RootfsGenUnitName(f.host, 3)}, {RoleWorker, 0, "jobs", WorkerUnitName(f.host, "jobs")}, {RoleCmd, 0, "", RootfsUnitName(RoleCmd, f.host)}} {
+ if err := c.StopRootfs(ctx, f.host, c2.role, c2.gen, c2.worker); err != nil {
+ t.Errorf("StopRootfs(%s, %d, %q): %v", c2.role, c2.gen, c2.worker, err)
+ }
+ f.ctl.mustHaveCall(t, "stop "+c2.unit)
+ }
+ inv, err = c.CmdRootfs(ctx, f.host, cgroup.TierChildren.MemoryMax, StagedImage{Path: f.imagePath, Version: rootfsVer, Digest: f.digest, Argv: []string{"bin/rails", "db:migrate:status"}})
+ if err != nil || len(inv) != 32 {
+ t.Fatalf("CmdRootfs: %q %v", inv, err)
+ }
+ f.ctl.set("show -p InvocationID,ActiveState,SubState,Result,ExecMainStatus,ExecMainCode "+RootfsUnitName(RoleCmd, f.host), "InvocationID="+inv+"\nActiveState=inactive\nSubState=dead\nResult=success\nExecMainStatus=0\nExecMainCode=exited\n")
+ if st, err := c.CmdStatusRootfs(ctx, f.host, inv); err != nil || st.State != CmdDone {
+ t.Errorf("CmdStatusRootfs = %+v, %v", st, err)
+ }
+ if st, err := c.CmdStatus(ctx, f.host, inv); err != nil || st.State != CmdGone {
+ t.Errorf("CmdStatus (the binary unit) = %+v, %v, want gone", st, err)
+ }
+ // Preflight 1.8: root's 403 is ErrStagedImage on the three launch verbs.
+ bad := StagedImage{Path: f.imagePath, Version: rootfsVer, Digest: strings.Repeat("0", 64), Process: release.ProcessWeb}
+ if _, err := c.StartRootfs(ctx, f.host, 0, cgroup.TierChildren.MemoryMax, bad); !errors.Is(err, ErrStagedImage) {
+ t.Errorf("StartRootfs with a bad digest: %v, want ErrStagedImage", err)
+ }
+ bad.Process = "jobs"
+ if _, err := c.StartRootfsWorker(ctx, f.host, "jobs", cgroup.TierChildren.MemoryMax, bad); !errors.Is(err, ErrStagedImage) {
+ t.Errorf("StartRootfsWorker with a bad digest: %v, want ErrStagedImage", err)
+ }
+ bad.Process = "release"
+ if err := c.StopRootfs(ctx, f.host, RoleCmd, 0, ""); err != nil {
+ t.Fatalf("StopRootfs(cmd): %v", err)
+ }
+ if _, err := c.CmdRootfs(ctx, f.host, cgroup.TierChildren.MemoryMax, bad); !errors.Is(err, ErrStagedImage) {
+ t.Errorf("CmdRootfs with a bad digest: %v, want ErrStagedImage", err)
+ }
+ // A 400 is not ErrStagedImage: the request was malformed, not refused.
+ if _, err := c.StartRootfs(ctx, f.host, 0, cgroup.TierChildren.MemoryMax, StagedImage{Path: f.imagePath, Version: "../v1", Digest: f.digest, Process: release.ProcessWeb}); err == nil || errors.Is(err, ErrStagedImage) {
+ t.Errorf("StartRootfs with a bad version: %v, want a plain error", err)
+ }
+ // ReadyRootfs: nil on 204, *NotReadyError carrying the report on 503
+ // (mirror TestClientReadyIsNilOn204AndCarriesTheReportOn503).
+ if err := c.ReadyRootfs(ctx); err != nil {
+ t.Errorf("ReadyRootfs with both probes green: %v", err)
+ }
+ f.srv.setRootfsReport(ProbeReport{Ready: false, Lines: []VerdictLine{{Check: "root-image", Verdict: "fail"}}})
+ var nr *NotReadyError
+ if err := c.ReadyRootfs(ctx); !errors.As(err, &nr) || len(nr.Report.Lines) == 0 || nr.Report.Lines[0].Check != "root-image" {
+ t.Errorf("ReadyRootfs with a failing rootfs probe: %v", err)
+ }
+ f.srv.setRootfsReport(ProbeReport{Ready: true})
+ c.Build = "other"
+ var stale *StaleBuildError
+ if err := c.ReadyRootfs(ctx); !errors.As(err, &stale) {
+ t.Errorf("ReadyRootfs from another build: %v, want StaleBuildError", err)
+ }
+}
+
+func TestRootfsDropInTexts(t *testing.T) {
+ host := "bill-2.example.com"
+ wantGen := `# Written by carlos instance-control for one generation; removed when it stops (zero-downtime plan 2).
+[Service]
+Slice=carlos-instances-bill\x2d2.example.com.slice
+BindPaths=/data/instances/bill-2.example.com /run/carlos/instances/bill-2.example.com
+ReadWritePaths=/data/instances/bill-2.example.com /run/carlos/instances/bill-2.example.com
+WorkingDirectory=/data/instances/bill-2.example.com
+RootImage=/run/carlos-control/exe/bill-2.example.com@7/rootfs/run/image
+BindReadOnlyPaths=/run/carlos-control/exe/bill-2.example.com@7/rootfs/run/launch.json:/opt/carlos/launch.json /run/carlos-control/exe/bill-2.example.com@7/rootfs/run/resolv.conf:/etc/resolv.conf
+StandardOutput=file:/run/carlos/logs/bill-2.example.com@7.run.sock
+MemoryMax=1073741824
+MemoryHigh=966367638
+MemorySwapMax=1073741824
+`
+ if got := RootfsGenDropIn(host, 7, cgroup.TierChildren); got != wantGen {
+ t.Errorf("RootfsGenDropIn =\n%s\nwant\n%s", got, wantGen)
+ }
+ wantWorker := `# Written by carlos instance-control for one rootfs worker; removed when it stops (rootfs apps plan 2).
+[Service]
+Slice=carlos-instances-bill\x2d2.example.com.slice
+BindPaths=/data/instances/bill-2.example.com /run/carlos/instances/bill-2.example.com
+ReadWritePaths=/data/instances/bill-2.example.com /run/carlos/instances/bill-2.example.com
+WorkingDirectory=/data/instances/bill-2.example.com
+RootImage=/run/carlos-control/exe/bill-2.example.com@jobs/rootfs/worker/image
+BindReadOnlyPaths=/run/carlos-control/exe/bill-2.example.com@jobs/rootfs/worker/launch.json:/opt/carlos/launch.json /run/carlos-control/exe/bill-2.example.com@jobs/rootfs/worker/resolv.conf:/etc/resolv.conf
+StandardOutput=file:/run/carlos/logs/bill-2.example.com@jobs.worker.sock
+`
+ if got := WorkerDropIn(host, "jobs", cgroup.TierChildren); got != wantWorker {
+ t.Errorf("WorkerDropIn =\n%s\nwant\n%s", got, wantWorker)
+ }
+ for _, body := range []string{wantGen, wantWorker} {
+ if strings.Count(body, "\nSlice=") != 1 || strings.Contains(body, "IPAddressAllow") {
+ t.Errorf("drop-in has not exactly one Slice= line, or an allow:\n%s", body)
+ }
+ }
+ if JITDropIn != "# Written by carlos instance-control for one start of a release that declares jit (rootfs apps design §5.11); removed when the unit stops.\n[Service]\nMemoryDenyWriteExecute=no\n" {
+ t.Errorf("JITDropIn = %q", JITDropIn)
+ }
+}
+
+// The hash a validation records is the file's, so a later image at the
+// same path with other bytes is never vouched for from the cache.
+func TestRootfsDigestOfAnotherFileAtTheSamePathIsRehashed(t *testing.T) {
+ f := rootfsFixture(t)
+ if _, err := f.srv.validateStagedImage(f.host, RoleRun, f.img(release.ProcessWeb)); err != nil {
+ t.Fatal(err)
+ }
+ other := []byte("a different image, renamed into place")
+ tmp := f.imagePath + ".tmp"
+ if err := os.WriteFile(tmp, other, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Rename(tmp, f.imagePath); err != nil {
+ t.Fatal(err)
+ }
+ sum := sha256.Sum256(other)
+ if _, err := f.srv.validateStagedImage(f.host, RoleRun, f.img(release.ProcessWeb)); !errors.Is(err, ErrStagedImage) {
+ t.Fatalf("a replaced image was vouched for: %v", err)
+ }
+ img := f.img(release.ProcessWeb)
+ img.Digest = hex.EncodeToString(sum[:])
+ // Its own digest, but the manifest's is the old one: still refused.
+ if _, err := f.srv.validateStagedImage(f.host, RoleRun, img); !errors.Is(err, ErrStagedImage) || !strings.Contains(err.Error(), "digest") {
+ t.Fatalf("an image the manifest does not name was vouched for: %v", err)
+ }
+}
diff --git a/internal/instancectl/server.go b/internal/instancectl/server.go
index 6ccff14ff..f46f0f580 100644
--- a/internal/instancectl/server.go
+++ b/internal/instancectl/server.go
@@ -7,10 +7,12 @@ import (
"errors"
"flag"
"fmt"
+ "io"
"log"
"net"
"net/http"
"net/netip"
+ "net/url"
"os"
"os/exec"
"path/filepath"
@@ -24,6 +26,7 @@ import (
"github.com/carlosframework/platform/internal/cliflag"
"github.com/carlosframework/platform/internal/controlsock"
"github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
"github.com/carlosframework/platform/internal/version"
)
@@ -70,6 +73,15 @@ type Server struct {
// (validateStagedExe). Either one unset refuses every staged start.
OptDir string
HostApp func(host string) (acct, app string, err error)
+ // HostRuntime reads a host's runtime from the registry (Route.Runtime).
+ // Only a row whose runtime is registry.RuntimeRootfsV1 is ever started
+ // from an image; nil refuses every rootfs start (rootfs apps plan 2).
+ HostRuntime func(host string) (string, error)
+ // ProbeImage writes the rootfs probe's squashfs at dst; nil skips the
+ // rootfs probe, which leaves the box not rootfs-ready (Task 7 fills it
+ // in). RootfsProbeOut is where the rootfs probe unit's output lands.
+ ProbeImage func(dst string) error
+ RootfsProbeOut string
// DropInDir is where generation units' runtime drop-ins go
// (/run/systemd/system). Empty refuses every generation start: the
// template alone runs nothing (zero-downtime plan 2 D1).
@@ -148,6 +160,21 @@ type Server struct {
// box not ready whatever the probe says, until an operator clears the
// files and restarts the service.
startLines []VerdictLine
+ // The rootfs probe's verdict (rootfs apps plan 2), beside the binary
+ // one; ReadyRootfs needs both.
+ rootfsReport ProbeReport
+ rootfsProbed bool
+ // The jit drop-in's owed-reload state (R11, rootfs.go): jitRemovals
+ // counts removals that deleted a file and post-start mismatches,
+ // jitReloadedThrough is the count a successful daemon-reload began
+ // after, and jitReloadedOnce is false until this service's first
+ // successful reload, because a restarted service cannot know what
+ // systemd has loaded.
+ jitRemovals, jitReloadedThrough uint64
+ jitReloadedOnce bool
+
+ // imageDigests caches validateStagedImage's hashes by file identity.
+ imageDigests imageDigests
}
const (
@@ -157,13 +184,21 @@ const (
)
// Run is `carlos instance-control`.
-func Run(args []string) { runWithClaim(args, nil, nil, nil) }
+func Run(args []string) { runWithClaim(args, nil, nil, RegistryView{}) }
+// RunWithRegistry is RunWithRegistryView with the account-and-app read only:
+// such a service refuses every rootfs start.
func RunWithRegistry(args []string, hostApp func(string) (string, string, error)) {
- runWithClaim(args, nil, nil, hostApp)
+ RunWithRegistryView(args, RegistryView{HostApp: hostApp})
+}
+
+// RunWithRegistryView serves with view as the service's reads of the
+// registry.
+func RunWithRegistryView(args []string, view RegistryView) {
+ runWithClaim(args, nil, nil, view)
}
-func runWithClaim(args []string, claim, heldRelease http.Handler, hostApp func(string) (string, string, error)) {
+func runWithClaim(args []string, claim, heldRelease http.Handler, view RegistryView) {
fs := flag.NewFlagSet("instance-control", flag.ExitOnError)
fs.Usage = cliflag.Usage(fs, "usage: carlos instance-control (socket-activated by carlos-instance-control.socket)")
_ = fs.Parse(args)
@@ -181,24 +216,28 @@ func runWithClaim(args []string, claim, heldRelease http.Handler, hostApp func(s
out, err := exec.CommandContext(ctx, args[0], args[1:]...).CombinedOutput()
return string(out), err
},
- PeerOK: productionPeerOK(),
- CgroupRoot: "/sys/fs/cgroup",
- ProbeOut: filepath.Join(controlsock.Dir, "probe.out"),
- FixtureOK: probeFixturesOK,
- ProbeEvery: 5 * time.Minute,
- Logf: log.Printf,
- ExeRoot: filepath.Join(controlsock.Dir, "exe"),
- OptDir: "/opt/carlos",
- HostApp: hostApp,
- DropInDir: "/run/systemd/system",
- ResolvConf: "/etc/resolv.conf",
- ResolvedConf: ResolvedConf,
- LocalAddrs: func() ([]netip.Prefix, error) { return localPrefixes(fibTriePath, ifInet6Path) },
- DataRoot: "/data/instances",
- Statfs: statfsBytes,
- DiskEvery: 30 * time.Second,
- Build: version.Build,
- ExeCurrent: installedBinaryCurrent,
+ PeerOK: productionPeerOK(),
+ CgroupRoot: "/sys/fs/cgroup",
+ ProbeOut: filepath.Join(controlsock.Dir, "probe.out"),
+ FixtureOK: probeFixturesOK,
+ ProbeEvery: 5 * time.Minute,
+ Logf: log.Printf,
+ ExeRoot: filepath.Join(controlsock.Dir, "exe"),
+ OptDir: "/opt/carlos",
+ HostApp: view.HostApp,
+ HostRuntime: view.HostRuntime,
+ // ProbeImage stays nil until the rootfs probe lands (Task 7), so a
+ // box running this build is not rootfs-ready.
+ RootfsProbeOut: filepath.Join(controlsock.Dir, "rootfs-probe.out"),
+ DropInDir: "/run/systemd/system",
+ ResolvConf: "/etc/resolv.conf",
+ ResolvedConf: ResolvedConf,
+ LocalAddrs: func() ([]netip.Prefix, error) { return localPrefixes(fibTriePath, ifInet6Path) },
+ DataRoot: "/data/instances",
+ Statfs: statfsBytes,
+ DiskEvery: 30 * time.Second,
+ Build: version.Build,
+ ExeCurrent: installedBinaryCurrent,
}
ln, activated, err := activatedListener()
if err != nil {
@@ -329,6 +368,7 @@ func (s *Server) Handler() http.Handler {
mux.HandleFunc("POST /cmd", s.claimAwareLegacy(s.handleCmd))
mux.HandleFunc("GET /cmd/{host}", s.claimAwareLegacy(s.handleCmdStatus))
mux.HandleFunc("GET /ready", s.handleReady)
+ mux.HandleFunc("GET "+RootfsReadyPath, s.handleReadyRootfs)
return mux
}
@@ -341,8 +381,20 @@ type request struct {
gen uint64 // a generation of the run role (zero-downtime spec §7.6); 0 is none
kill bool
exe StagedExe // a staged build to mount (zero-downtime plan 3); Path "" is none
+ // runtime is "rootfs" when the verb is about a rootfs unit (rootfs apps
+ // plan 2), "" otherwise; image is the staged image a rootfs launch
+ // names, and worker the worker name a RoleWorker request carries.
+ runtime string
+ worker string
+ image StagedImage
}
+// rootfs reports whether the request is about a rootfs unit.
+func (req request) rootfs() bool { return req.runtime == runtimeRootfs }
+
+// runtimeRootfs is the one value the runtime query field takes.
+const runtimeRootfs = "rootfs"
+
// verbShape is what a verb's query may carry.
type verbShape struct {
roles []Role
@@ -356,19 +408,80 @@ type verbShape struct {
// cmd is /cmd, whose staged build is the serving generation's, as the
// legacy sidecar's is (run spec §5.4 under zero-downtime spec §7.5).
cmd bool
+ // rootfs is a launch verb (/start, /cmd): runtime=rootfs and the image
+ // fields come together or not at all. rootfsStop is a termination verb
+ // or the status poll (/stop, /reset, GET /cmd/{host}): runtime=rootfs
+ // alone, and any image field is a 400 (Astra R1 finding 6). worker
+ // admits worker=<name> beside role=worker.
+ rootfs bool
+ rootfsStop bool
+ worker bool
}
+// maxCmdBody bounds the JSON body of a rootfs /cmd ({"argv": [...]}).
+const maxCmdBody = 64 << 10
+
func parseRequest(r *http.Request, v verbShape) (request, error) {
q := r.URL.Query()
host := q.Get("host")
if !provision.ValidHostShape(host) {
return request{}, fmt.Errorf("host %q is not a host", host)
}
+ req, err := parseQuery(host, q, v)
+ if err != nil {
+ return request{}, err
+ }
+ if v.cmd && req.rootfs() && r.Body != nil {
+ // An operator's argv rides in the body, bounded; the query names
+ // the process instead, never both (Astra R1 finding 7).
+ raw, err := io.ReadAll(http.MaxBytesReader(nil, r.Body, maxCmdBody))
+ if err != nil {
+ return request{}, fmt.Errorf("cmd body: %v", err)
+ }
+ if len(raw) > 0 {
+ var body struct {
+ Argv []string `json:"argv"`
+ }
+ if err := json.Unmarshal(raw, &body); err != nil {
+ return request{}, fmt.Errorf("cmd body: %v", err)
+ }
+ req.image.Argv = body.Argv
+ }
+ if (req.image.Process == "") == (len(req.image.Argv) == 0) {
+ return request{}, errors.New("a rootfs cmd names a process or carries an argv, one of the two")
+ }
+ for _, w := range req.image.Argv {
+ if w == "" || strings.ContainsRune(w, 0) {
+ return request{}, errors.New("argv holds an empty word or a NUL byte")
+ }
+ }
+ }
+ return req, nil
+}
+
+// parseQuery is parseRequest over the query alone, for a verb whose host
+// is in its path (GET /cmd/{host}).
+func parseQuery(host string, q url.Values, v verbShape) (request, error) {
req := request{host: host}
+ // role=worker is a rootfs role only (preflight 1.1): without
+ // runtime=rootfs it is refused on every verb before any handler runs, or
+ // POST /stop?role=worker would name the host's binary run unit.
+ if (q.Get("role") == string(RoleWorker) || q.Has("worker")) && q.Get("runtime") != runtimeRootfs {
+ return request{}, errors.New("role worker is a rootfs role: it needs runtime=rootfs")
+ }
+ if q.Has("runtime") {
+ if q.Get("runtime") != runtimeRootfs {
+ return request{}, fmt.Errorf("runtime %q is not rootfs", q.Get("runtime"))
+ }
+ if !v.rootfs && !v.rootfsStop {
+ return request{}, errors.New("this verb takes no runtime")
+ }
+ req.runtime = runtimeRootfs
+ }
if len(v.roles) > 0 {
role, ok := ParseRole(q.Get("role"))
if !ok {
- return request{}, fmt.Errorf("role %q is not run, seed, sidecar or cmd", q.Get("role"))
+ return request{}, fmt.Errorf("role %q is not run, seed, sidecar, cmd or worker", q.Get("role"))
}
allowed := false
for _, r := range v.roles {
@@ -379,6 +492,42 @@ func parseRequest(r *http.Request, v verbShape) (request, error) {
}
req.role = role
}
+ if q.Has("worker") {
+ name := q.Get("worker")
+ if !v.worker || req.role != RoleWorker {
+ return request{}, errors.New("worker=<name> belongs to role=worker on a rootfs verb")
+ }
+ if !release.ValidWorkerName(name) {
+ return request{}, fmt.Errorf("worker %q is not a worker name", name)
+ }
+ req.worker = name
+ }
+ if req.role == RoleWorker && req.worker == "" {
+ return request{}, errors.New("role worker needs worker=<name>")
+ }
+ if q.Has("image") || q.Has("process") {
+ if !v.rootfs {
+ return request{}, errors.New("this verb takes no image, version, digest or process")
+ }
+ if !req.rootfs() {
+ return request{}, errors.New("an image needs runtime=rootfs")
+ }
+ }
+ if req.rootfs() && v.rootfsStop && (q.Has("version") || q.Has("digest")) {
+ return request{}, errors.New("this verb takes no image, version, digest or process")
+ }
+ if req.rootfs() && v.rootfs {
+ if !q.Has("image") || q.Get("image") == "" {
+ return request{}, errors.New("runtime=rootfs needs an image, its version and its digest")
+ }
+ if q.Has("exe") {
+ return request{}, errors.New("a rootfs launch names an image, never an exe")
+ }
+ if !release.ValidVersion(q.Get("version")) {
+ return request{}, fmt.Errorf("version %q is not a version", q.Get("version"))
+ }
+ req.image = StagedImage{Path: q.Get("image"), Version: q.Get("version"), Digest: q.Get("digest"), Process: q.Get("process")}
+ }
if v.tier {
n, err := strconv.ParseInt(q.Get("memory_max"), 10, 64)
if err != nil {
@@ -411,7 +560,9 @@ func parseRequest(r *http.Request, v verbShape) (request, error) {
}
req.kill = true
}
- if q.Has("exe") || q.Has("version") || q.Has("digest") {
+ // A rootfs launch's version and digest are its image's (above); every
+ // other verb's are a staged executable's.
+ if q.Has("exe") || (!req.rootfs() && (q.Has("version") || q.Has("digest"))) {
if !v.exe {
return request{}, errors.New("this verb takes no staged executable")
}
@@ -459,16 +610,27 @@ func writeJSON(w http.ResponseWriter, status int, v any) {
}
func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) {
- req, err := parseRequest(r, verbShape{roles: []Role{RoleRun, RoleSidecar}, tier: true, gen: true, exe: true})
+ req, err := parseRequest(r, verbShape{roles: []Role{RoleRun, RoleSidecar, RoleWorker}, tier: true, gen: true, exe: true, rootfs: true, worker: true})
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
+ if req.rootfs() {
+ s.startRootfs(w, r, req)
+ return
+ }
if req.gen != 0 {
s.startGen(w, r, req)
return
}
ctx := r.Context()
+ unit := UnitName(req.role, req.host)
+ if unit == "" {
+ // Only a worker has no binary unit, and parseRequest refused it
+ // above; this stands so no later role can reach systemctl as "".
+ http.Error(w, "role "+string(req.role)+" has no binary unit", http.StatusBadRequest)
+ return
+ }
if req.role == RoleRun {
// An attempt a /stop of this host can cancel and join, and that a
// stop which has already run refuses (legacystart.go, #292).
@@ -480,7 +642,6 @@ func (s *Server) handleStart(w http.ResponseWriter, r *http.Request) {
defer finish()
ctx = attempt
}
- unit := UnitName(req.role, req.host)
// A sidecar naming its generation's staged build, or a run unit naming
// the build its cold wake recorded, is validated before anything is
// laid out, under the generation rule (validateStagedExe).
@@ -658,15 +819,24 @@ func (s *Server) stopUnit(ctx context.Context, unit string) {
}
func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) {
- req, err := parseRequest(r, verbShape{roles: []Role{RoleRun, RoleSeed, RoleSidecar, RoleCmd}, gen: true, kill: true})
+ req, err := parseRequest(r, verbShape{roles: []Role{RoleRun, RoleSeed, RoleSidecar, RoleCmd, RoleWorker}, gen: true, kill: true, rootfsStop: true, worker: true})
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
}
+ if req.rootfs() {
+ s.stopRootfs(w, r, req)
+ return
+ }
if req.gen != 0 {
s.stopGen(w, r, req)
return
}
+ unit := UnitName(req.role, req.host)
+ if unit == "" {
+ http.Error(w, "role "+string(req.role)+" has no binary unit", http.StatusBadRequest)
+ return
+ }
if req.role == RoleRun {
// Every start of this host's run unit in flight is cancelled and
// joined first, and earlier requests' starts are refused from here
@@ -677,12 +847,12 @@ func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) {
defer release()
if err != nil {
// Stopped anyway, but not proven final: a start may follow.
- _, _ = s.Systemctl(r.Context(), "stop", UnitName(req.role, req.host))
+ _, _ = s.Systemctl(r.Context(), "stop", unit)
http.Error(w, "stop: "+err.Error(), http.StatusBadGateway)
return
}
}
- if out, err := s.Systemctl(r.Context(), "stop", UnitName(req.role, req.host)); err != nil {
+ if out, err := s.Systemctl(r.Context(), "stop", unit); err != nil {
http.Error(w, "stop: "+strings.TrimSpace(out+" "+err.Error()), http.StatusBadGateway)
return
}
@@ -693,13 +863,13 @@ func (s *Server) handleStop(w http.ResponseWriter, r *http.Request) {
}
}
if req.role == RoleCmd {
- s.cleanCmdExe(r.Context(), req.host)
+ s.cleanCmdExe(r.Context(), req.host, false)
}
w.WriteHeader(http.StatusNoContent)
}
func (s *Server) handleReset(w http.ResponseWriter, r *http.Request) {
- req, err := parseRequest(r, verbShape{roles: []Role{RoleRun, RoleSeed, RoleSidecar, RoleCmd}, gen: true})
+ req, err := parseRequest(r, verbShape{roles: []Role{RoleRun, RoleSeed, RoleSidecar, RoleCmd, RoleWorker}, gen: true, rootfsStop: true, worker: true})
if err != nil {
http.Error(w, err.Error(), http.StatusBadRequest)
return
@@ -708,6 +878,13 @@ func (s *Server) handleReset(w http.ResponseWriter, r *http.Request) {
if req.gen != 0 {
unit = GenUnitName(req.host, req.gen)
}
+ if req.rootfs() {
+ unit = rootfsUnitOf(req)
+ }
+ if unit == "" {
+ http.Error(w, "role "+string(req.role)+" has no binary unit", http.StatusBadRequest)
+ return
+ }
if out, err := s.Systemctl(r.Context(), "reset-failed", unit); err != nil && !strings.Contains(out+err.Error(), "not loaded") {
http.Error(w, "reset-failed: "+strings.TrimSpace(out+" "+err.Error()), http.StatusBadGateway)
return
@@ -723,7 +900,8 @@ func (s *Server) handleSweep(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
targets := map[string]bool{}
out, err := s.Systemctl(ctx, "list-units", "--all", "--plain", "--no-legend", "--no-pager",
- UnitPrefixRun+"*.service", UnitPrefixSeed+"*.service", UnitPrefixSidecar+"*.service", UnitPrefixCmd+"*.service", UnitPrefixGen+"*.service")
+ UnitPrefixRun+"*.service", UnitPrefixSeed+"*.service", UnitPrefixSidecar+"*.service", UnitPrefixCmd+"*.service", UnitPrefixGen+"*.service",
+ UnitPrefixRootfs+"*.service", UnitPrefixRootfsGen+"*.service", UnitPrefixRootfsWorker+"*.service", UnitPrefixRootfsCmd+"*.service")
if err != nil {
http.Error(w, "list-units: "+err.Error(), http.StatusBadGateway)
return
@@ -784,6 +962,10 @@ func (s *Server) handleSweep(w http.ResponseWriter, r *http.Request) {
continue
}
stopped = append(stopped, unit)
+ if IsRootfsUnit(unit) {
+ s.removeRootfsFiles(unit)
+ continue
+ }
s.removeGenFiles(unit)
if role, esc, _ := ParseUnitName(unit); role == RoleCmd {
if iid, ok := Unescape(esc); ok {
@@ -981,6 +1163,7 @@ func (s *Server) RunProbe(ctx context.Context) {
}
}
set(s.withStartLines(r))
+ s.runRootfsProbe(ctx)
}
// mountpoints are the two bind mounts whose underlying directories
diff --git a/internal/instancectl/server_gen_test.go b/internal/instancectl/server_gen_test.go
index 577b5baf1..0fa9a2d96 100644
--- a/internal/instancectl/server_gen_test.go
+++ b/internal/instancectl/server_gen_test.go
@@ -528,7 +528,9 @@ func TestSweepRemovesAnExeOnlyGenerationLeftover(t *testing.T) {
if err := s.ensureHostExe(genHost); err != nil {
t.Fatal(err)
}
- odd := []string{genHost + "@0", genHost + "@x", "@7", genHost + "@07"}
+ // Neither generation keys nor worker keys ("@x" would be a worker's,
+ // rootfs apps plan 2, so the letter is one no worker name takes).
+ odd := []string{genHost + "@0", genHost + "@X", "@7", genHost + "@07"}
for _, name := range odd {
if err := os.Mkdir(filepath.Join(s.ExeRoot, name), 0o755); err != nil {
t.Fatal(err)
diff --git a/internal/instancectl/server_test.go b/internal/instancectl/server_test.go
index c2e12c2b3..ddab839bd 100644
--- a/internal/instancectl/server_test.go
+++ b/internal/instancectl/server_test.go
@@ -35,6 +35,11 @@ type fakeCtl struct {
// (#292): its systemctl calls run although the request's context is
// done, the race exec.CommandContext cannot close on its own.
unseenCancel bool
+ // dynamic, if set, is consulted after fail and before answers: a model
+ // of systemd's state that changes with the calls it sees (the rootfs
+ // fixture's invocations and loaded units). It answers (out, true) or
+ // defers with false.
+ dynamic func(args []string) (string, bool)
}
func (f *fakeCtl) run(ctx context.Context, args ...string) (string, error) {
@@ -68,19 +73,57 @@ func (f *fakeCtl) run(ctx context.Context, args ...string) (string, error) {
if ctx.Err() != nil && !unseen {
return "", ctx.Err()
}
+ // The longest matching key wins in both maps (preflight 2.12): a test
+ // may set "show -p InvocationID" beside "show -p InvocationID,Slice,...",
+ // and Go's map order must not pick between them. A failing call still
+ // returns the matching answer as its output, as systemctl prints its
+ // reason beside a non-zero exit.
f.mu.Lock()
- defer f.mu.Unlock()
- for k, err := range f.fail {
- if strings.HasPrefix(line, k) {
- return "", err
+ failKey, failed := longestPrefix(f.fail, line)
+ answerKey, answered := longestPrefix(f.answers, line)
+ var out string
+ if answered {
+ out = f.answers[answerKey]
+ }
+ var err error
+ if failed {
+ err = f.fail[failKey]
+ }
+ dynamic := f.dynamic
+ f.mu.Unlock()
+ if failed {
+ return out, err
+ }
+ if dynamic != nil {
+ if dyn, ok := dynamic(args); ok {
+ return dyn, nil
+ }
+ }
+ return out, nil
+}
+
+// longestPrefix returns the longest key of m that is a prefix of line.
+func longestPrefix[V any](m map[string]V, line string) (string, bool) {
+ var best string
+ var found bool
+ for k := range m {
+ if strings.HasPrefix(line, k) && (!found || len(k) > len(best)) {
+ best, found = k, true
}
}
- for k, out := range f.answers {
+ return best, found
+}
+
+// hasAnswer reports whether an explicit answer is set for line.
+func (f *fakeCtl) hasAnswer(line string) bool {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ for k := range f.answers {
if strings.HasPrefix(line, k) {
- return out, nil
+ return true
}
}
- return "", nil
+ return false
}
func (f *fakeCtl) joined() string {