5f9186b21 · carlos/platform

diff --git a/cli.go b/cli.go

index b2ccc2f49..0e2277fd7 100644

--- a/cli.go

+++ b/cli.go

@@ -80,6 +80,29 @@ func refuseAddNested(host string, existing *registry.Route, dataDir string) erro

return nil

}

+// addMintsNested reports whether `carlos add` on this box mints the

+// nested defaults: the box's layout intent names nested. With no deploy

+// store in the environment (a box before its bootstrap, a dev shell) the

+// answer is flat, as it always was. An intent that cannot be read is an

+// error, not a guess: a flat row on a nested box is the thing this exists

+// to prevent, and the operator can pass --socket and --db to say so

+// explicitly.

+func addMintsNested(ctx context.Context, store release.Store, boxLabel func() string) (bool, error) {

+ if store == nil {

+ return false, nil

+ }

+ in, found, err := release.LoadLayoutIntent(ctx, store, boxLabel())

+ if err != nil {

+ return false, fmt.Errorf("could not read the box's layout intent, so the row's paths cannot be chosen: %v. Retry, or pass --socket and --db", err)

+ }

+ return found && in.Mode == release.LayoutModeNested, nil

+}

+

+// boxLabelForAdd is the label this box's status publishes under, which is

+// what a layout intent's scope names. A function value so runAdd's `host`

+// string does not shadow the package.

+var boxLabelForAdd = func() string { return host.BoxLabel() }

+

func runAdd(args []string) {

fs := flag.NewFlagSet("add", flag.ExitOnError)

reg := fs.String("registry", defaultRegistry(), "registry database")

@@ -153,6 +176,22 @@ func runAdd(args []string) {

if *dbPath == "" && *kind == "instance" && *backing != registry.BackingUnit {

*dbPath = provision.DefaultDBPath(dataDir(), host)

}

+ // On a box whose layout intent is nested, a row this verb would make

+ // activator-managed gets the nested defaults (design §8.6 step 5,

+ // plan 5): it starts out isolated instead of running flat for a pass

+ // and being moved. Explicit --socket or --db is the operator's own

+ // choice and is left alone (the pass reports it foreign).

+ if *kind == "instance" && *backing == "" && *hibernate && *addr == "" &&

+ *socket == provision.DefaultSocket(host) && *dbPath == provision.DefaultDBPath(dataDir(), host) {

+ store, _ := deploymentReadSeam()

+ nested, err := addMintsNested(context.Background(), store, boxLabelForAdd)

+ if err != nil {

+ log.Fatalf("add %s: %v", host, err)

+ }

+ if nested {

+ *socket, *dbPath = provision.NestedSocket(host), provision.NestedDBPath(dataDir(), host)

+ }

// A hibernating unit tenant stops and starts with systemd, not the

// agent — box-wide litestream (root) is the ONLY thing that can

// replicate it, so skipping --db here is a genuine data-loss risk

diff --git a/cli_test.go b/cli_test.go

index a2a499472..85ec6fa44 100644

--- a/cli_test.go

+++ b/cli_test.go

@@ -1399,3 +1399,46 @@ func TestRefuseAddNested(t *testing.T) {

t.Fatalf("refuseAddNested(flat) = %v, want nil", err)

}

+

+// `carlos add` mints the nested defaults only when the box's layout

+// intent names nested; no store means flat as before, and an unreadable

+// intent is a refusal, not a guess.

+func TestAddMintsNestedOnlyUnderANestedIntent(t *testing.T) {

+ ctx := context.Background()

+ label := func() string { return "kass-1" }

+ if nested, err := addMintsNested(ctx, nil, label); err != nil || nested {

+ t.Fatalf("no store = %v %v, want flat", nested, err)

+ }

+ s := release.NewDirStore(t.TempDir())

+ if nested, err := addMintsNested(ctx, s, label); err != nil || nested {

+ t.Fatalf("no intent = %v %v, want flat", nested, err)

+ }

+ if _, err := release.AppendLayoutIntent(ctx, s, release.LayoutIntent{Mode: release.LayoutModeNested, Scope: release.LayoutScope{Box: "kass-2"}}); err != nil {

+ t.Fatal(err)

+ }

+ if nested, err := addMintsNested(ctx, s, label); err != nil || nested {

+ t.Fatalf("another box's intent = %v %v, want flat", nested, err)

+ }

+ if _, err := release.AppendLayoutIntent(ctx, s, release.LayoutIntent{Mode: release.LayoutModeNested, Scope: release.LayoutScope{Box: "kass-1"}}); err != nil {

+ t.Fatal(err)

+ }

+ if nested, err := addMintsNested(ctx, s, label); err != nil || !nested {

+ t.Fatalf("this box's nested intent = %v %v, want nested", nested, err)

+ }

+ if _, err := release.AppendLayoutIntent(ctx, s, release.LayoutIntent{Mode: release.LayoutModeFlat}); err != nil {

+ t.Fatal(err)

+ }

+ if nested, err := addMintsNested(ctx, s, label); err != nil || nested {

+ t.Fatalf("a later fleet-wide flat intent = %v %v, want flat", nested, err)

+ }

+ // Unreadable (a newer schema): refused with the retry named.

+ if err := s.Delete(ctx, release.LayoutIntentKey()); err != nil {

+ t.Fatal(err)

+ }

+ if _, err := s.Put(ctx, release.LayoutIntentKey(), []byte(`{"schema":2,"requests":[]}`), ""); err != nil {

+ t.Fatal(err)

+ }

+ if _, err := addMintsNested(ctx, s, label); err == nil || !strings.Contains(err.Error(), "pass --socket and --db") {

+ t.Fatalf("unreadable intent = %v, want a refusal naming the flags", err)

+ }

+}

diff --git a/docs/site/cli.md b/docs/site/cli.md

index 4656ac7df..ff2be4626 100644

--- a/docs/site/cli.md

+++ b/docs/site/cli.md

@@ -1706,7 +1706,7 @@ list is for people who roll releases without owning the account. It covers

console signs an uploaded build with the deployment key and every box runs it,

so only an owner can ship one.

-A box that has isolated instances refuses to install a platform release from before the instance layout, whether by update or rollback, until its layout has been reverted and every instance reports `flat`.

+A box that has isolated instances refuses to install a platform release from before the instance layout, whether by update or rollback, until its layout has been reverted and every instance reports `flat`. The box judges the candidate by the units it ships and, when the candidate's version is a release number, by that number against the first release on which isolated instances run; a locally built binary named by a commit is judged by its units alone. The console refuses the same move before it reaches a box: `update` to a named release from before the layout, or `rollback` onto one, is refused while any box reports an instance that is `nested`, `fenced` or `conflict`, or has not reported its layout at all.

```sh

carlos system update --channel canary

@@ -1715,6 +1715,24 @@ carlos system update --channel stable --spread 10m

carlos system rollback --channel stable

```

+#### Instance layout

+

+Every instance the activator runs used to live in one shared directory as one user, where it could read its neighbours' files. `layout migrate` moves them out: each instance gets its own directory and, from its next wake, its own mount and pid namespace, where it sees its own files and nothing else. An instance that is asleep is moved at once; one that is awake is left for up to an hour to go to sleep, then stopped briefly and moved. `--force` skips the hour. `--box` scopes the request to one box, so a fleet moves one box at a time with `carlos status --layout` showing each box's rows turn `nested`. Once a box's layout is `nested`, a newly provisioned instance on it starts out `nested`; it is never created in the shared directory first.

+

+`layout migrate` is refused while any box in scope runs a platform release from before the layout, so a fleet is rolled forward first and migrated second.

+

+`layout revert` moves them back. The order for rolling the platform back to a release from before the layout is: `layout revert`, wait until `carlos status --layout` shows every instance `flat`, then `system rollback`. The console will not move a channel pointer onto such a release while any box reports otherwise, and a revert is refused while a maintenance worker is attached to a box in scope. `layout status` prints the recorded intent; what each box has done is `carlos status --layout`.

+

+```sh

+carlos system layout migrate --box kass-1

+carlos status --layout

+carlos system layout migrate

+carlos system layout revert --force

+carlos system layout status

+```

+

+Ops account owners and `CARLOS_SYSTEM_OPERATORS`, like `update` and `rollback`.

+

A box is put on canary with `CARLOS_SYSTEM_CHANNEL=canary` in its host.env.

Boxes that have not reported a platform yet count as stable. `carlos system

apply` is the root oneshot behind `carlos-system-update.service`; it is not

diff --git a/docs/superpowers/specs/2026-09-11-isolated-exec-design.md b/docs/superpowers/specs/2026-09-11-isolated-exec-design.md

index cebaa53b7..b1eba239e 100644

--- a/docs/superpowers/specs/2026-09-11-isolated-exec-design.md

+++ b/docs/superpowers/specs/2026-09-11-isolated-exec-design.md

@@ -1,6 +1,6 @@

# Isolated exec — every exec-backed instance in its own namespace

-**Date:** 2026-09-11 · **Status:** revision 15; codex verdict after

+**Date:** 2026-09-11 · **Status:** revision 16; codex verdict after

round 11 is **accept with corrections**, and the one correction is

applied here (54 findings over 11 rounds, all accepted; §13); a

whole-branch final review (plan 2) found six further items, fixed in

@@ -1658,8 +1658,8 @@ of `flat`, `nested`, `fenced` (with the journal's phase and reason),

`conflict` (with the other host), `foreign` (with both values),

`external`, and a box-level

`layout_mode` (`flat`, `nested`, or `none` before any intent). `carlos

-status`, `carlos instances --layout` and the console's instance page

-render them; a box that predates the field shows `unknown (box predates

+status --layout` (beside `--instances`, under the same operator gate;

+R14) and the console's instance page render them; a box that predates the field shows `unknown (box predates

layout reporting)`. It is the pre-roll inventory, the progress view

during a migration, the member-visible proof that an instance is

isolated (`nested`), and the reason a `fenced` host does not wake.

@@ -1762,7 +1762,12 @@ fails only on a line that names the property that is missing.

for flagged routes (and keeps them flat for unflagged ones, whose

supervisors own their paths), removes the spawn half of

`internal/cgroup` and `Delegate=` on the edge, and refuses to start on

- a box whose `layout_mode` is not `nested`.

+ a box whose `layout_mode` is not `nested`. Brought forward in plan 5

+ (revision 16): on a box whose intent is `nested`, the reconciler and

+ `carlos add` mint a fresh activator-managed row at the nested paths

+ (`routeFor`'s `nested` argument), so a new tenant never runs flat

+ there, not even for one pass. `DefaultDBPath`/`DefaultSocket` stay

+ flat for every other caller until the next release.

**The roll script and the planner.** `update-platform-binary.sh:52-70`

(pinfra) enables units by name; it gains `carlos-instance-control.socket`,

@@ -2195,5 +2200,6 @@ rollback runs the old binary).

R11-1 the probe fixture comes from tmpfiles at bootstrap, and the control service verifies its ownership and mode before accepting a verdict (§3.2, §8.5).

- R12 (final review, plan 2): cgroup-ro targets the edge's delegated cgroup.procs; DAC precedes the mount flag so a root-owned file answers EACCES.

- R13 (task 4 review, plan 3): after a stop the activator requested, a unit reporting a foreign or absent invocation proves the old invocation over without the cgroup gate (§6.5); the observer never stops an invocation it did not start once its own stop was requested.

+- R14 (plan 4, 2026-10-04; recorded in revision 16): the per-host layout view is `carlos status --layout`, beside `--instances` and under the same operator gate, not `carlos instances --layout`: the instances verb lists the local registry, and the layout report lives in each box's private status snapshot, which only the console reads (§8.4).

- R15 (first roll, 2026-09-12; fixed 2026-09-25): the probe on dev's boxes answered `ready: false` twice, and the review of the fix found a third gap. (a) `RestrictSUIDSGID=yes` makes `openat2` fail with ENOSYS on systemd 257, because systemd cannot read the mode inside `open_how` and so blocks the whole call; every `instancefs` open in the namespace is an `openat2`. The line is dropped from all four templates. What it guarded is covered another way: with no capabilities an instance can only make a set-id file owned by `carlos`, which gains nothing for another `carlos` process; `NoNewPrivileges=yes` voids it inside the namespace; and no other uid can reach one (the tenant trees are `nosuid`, and `PrivateTmp=`'s `/tmp` and `/var/tmp` sit under a root 0700 directory, `/var/tmp` being on the root filesystem and not `nosuid`). (b) `control-unreachable` checked `Lstat`, but `InaccessiblePaths=` leaves a mode 000 root-owned node at the path. The check is now what a tenant could do, a listing of the directory and a dial of the socket, and passes only when both are refused for permission or absence; a refused connection or a timeout proves the path reachable and fails. (c) `PrivateDevices=` bind-mounts the host's `/dev/shm`, so every instance, all uid `carlos`, shared one; each unit now gets its own tmpfs there, and the probe's `shm-private` check plants a file in the host's `/dev/shm` (the control service's tmpfiles fixture) and fails if it is visible. POSIX message queues and abstract unix sockets are still box-wide; they are a follow-up (§11). (d) 2026-10-04, plan 4: `carlos-edge.service` carried the same line, and the agent runs there: its nested `instancefs` opens and every rename of the layout pass are `openat2` calls, so the first `layout migrate` would have met ENOSYS on its first host. The line is dropped from the edge unit too. The argument in (a) carries: the edge's one capability, `CAP_NET_BIND_SERVICE`, does not bear on set-id files (that takes `CAP_CHOWN`, `CAP_FSETID` or `CAP_SETUID`), so anything the agent could mark set-id is owned by `carlos`, and the agent and its flat children run under `NoNewPrivileges=`. An `openat` fallback in `instancefs` was considered and rejected: it would be a second resolution path, exercised in production only when the hardening had regressed, and it would route around the regression rather than report it. Instead the agent runs `instancefs.SelfCheck` in its own unit at start, and the nested-wake gate (§8.5) asks it first, so a blocked `openat2` refuses nested wakes and the layout pass with the directive named.

- R16 (plan 4 review, 2026-10-04): three gaps in the pass while flat tenants still share the agent's uid. (a) A tenant could make its own future directory under `/data/.instances` ahead of the pass and hard-link a sibling's database or `litestream.env` into it (its own uid's files, so `protected_hardlinks` allows it); `EnsureDirs` accepted the directory and the destination check looked only at the planned names. Now a migration refuses an instance directory holding any entry the plan does not name, refuses any file about to move that has more than one name (the shadow directory walked), and checks both again after the renames and before the row flips; a failure there keeps the journal at `files` and the host fenced. (b) Deferring an awake host (plan 4 amendment, "when it is asleep") was unbounded, so a tenant that never idled kept raw-path access to its nested siblings for good. The deferral is bounded: an hour of grace (`layout.DefaultAwakeGrace`), after which the pass stops the host as §8.2 step 2 always allowed and moves it; the serial leg re-runs on its own ten minutes after any deferral so the bound holds on a quiet fleet; the intent gains `force` for an operator who wants no grace; and `carlos status --layout` warns on every box with nested rows beside flat ones, because that box is not isolated yet. (c) Files moved aside at a destination (`<name>.aside-<unix>`) were kept inside the instance directory, readable by the tenant that gets the directory; they are kept under `/data/layout/aside` now, which no instance namespace contains. Still open: a flat tenant that is running while another host moves can plant between the check and the rename; the check is against what was planted before the pass arrived, and the closure is the bound in (b). (d) Re-check: (a)'s refusals were themselves escapable: a tenant could keep its own move refused with one syscall (`ln /data/B.db /data/.x`, or a file in its own future directory), and because the pass lifted its fence on a refusal the host woke flat again on its next request, for as long as it kept the refusal alive. The rule is now fail-closed on the way to nested: a flat host that has been stopped for its move and is then refused stays fenced and down (the pass keeps the fence, as it already did after a failed handover), status reports it `fenced` with the refusal (`held down after a refused move to nested`), every later pass retries from the kept fence and releases the host once it has moved, and a flat intent lifts the fence. A nested host refused its revert is not held: it is already isolated. The mixed-layout warning counts `foreign` and `external` rows beside `flat` ones, since all three run as uid carlos with no namespace; and the second-name refusal reports the inode and the `find <data> -xdev -samefile <path>` that lists the other names. (e) Landing follow-ups: a planted ENTRY in a host's future directory is no longer a refusal (one `touch` by a sibling would have held the host down until an operator acted, and the hold outlived the sibling's own migration); it is renamed into `/data/layout/aside/<host>/` with a random, no-replace name and the move goes on, logged and shown in status as a note. The hold stays for a second name (`nlink > 1`, no safe automatic fix) and for a symlink at the instance directory. The hold is persisted as `/data/layout/<host>.hold` and restored at agent start before anything wakes, by re-deriving it from the files: the marker is a hint to re-check, not the authority, because every file under `/data` is uid carlos's and a flat tenant can unlink it; a deleted marker therefore costs one grace after one restart, after which the next pass meets the same refusal and holds again, and a marker whose reason is gone is dropped rather than honoured. The serial leg re-runs on its own after a held result, as it does after a deferral, so a cleared reason is acted on within ten minutes on a quiet fleet.

diff --git a/internal/console/apistatuslayout.go b/internal/console/apistatuslayout.go

index 96244652b..8623edc1c 100644

--- a/internal/console/apistatuslayout.go

+++ b/internal/console/apistatuslayout.go

@@ -39,21 +39,25 @@ type apiStatusLayoutBox struct {

Mode string `json:"mode"`

Stale bool `json:"stale"`

Rows []apiStatusLayoutRow `json:"rows"`

- // Mixed is set when the box has a nested row beside a flat, foreign

- // or external one. Each of those runs as the agent's uid in the

+ // Mixed is set when the box has a flat, foreign or external row

+ // beside a nested one, or under a nested intent. Each of those runs as the agent's uid in the

// agent's mount namespace, so it reaches every nested sibling's raw

// directory: the nested rows are not isolated until none remains on

// the box. Additive; an old CLI ignores it.

Mixed bool `json:"mixed,omitempty"`

}

-// layoutMixed is the Mixed rule: at least one nested row beside at least

-// one row that runs as the agent's uid with no namespace, which is a flat

-// row, a foreign one (paths the pass leaves alone, spawned the flat way)

-// or an external one (another supervisor's process, also uid carlos).

+// layoutMixed is the Mixed rule: a row that runs as the agent's uid with

+// no namespace, which is a flat row, a foreign one (paths the pass leaves

+// alone, spawned the flat way) or an external one (another supervisor's

+// process, also uid carlos), on a box that is nested, either by a nested

+// row beside it or by the box's intent (mode) being nested. The second

+// half (plan 5) keeps the warning loud for a box that has gone nested but

+// whose only remaining rows are the ones the pass cannot move: with no

+// nested row left to stand beside, they would otherwise read as clean.

// Fenced and conflict rows are stopped and count for neither side.

-func layoutMixed(rows []apiStatusLayoutRow) bool {

- nested, unisolated := false, false

+func layoutMixed(mode string, rows []apiStatusLayoutRow) bool {

+ nested, unisolated := mode == "nested", false

for _, r := range rows {

switch r.Layout {

case "nested":

@@ -109,7 +113,7 @@ func (s *Server) handleAPIStatusLayout(w http.ResponseWriter, r *http.Request) {

Mode: b.Metrics.Layout.Mode,

Stale: status.BoxStale(b, now, status.SnapshotPeriod),

Rows: rows,

- Mixed: layoutMixed(rows),

+ Mixed: layoutMixed(b.Metrics.Layout.Mode, rows),

})

}

sort.Slice(out, func(i, j int) bool { return out[i].Box < out[j].Box })

diff --git a/internal/console/apistatuslayout_test.go b/internal/console/apistatuslayout_test.go

index fe3459c17..0d2f383bc 100644

--- a/internal/console/apistatuslayout_test.go

+++ b/internal/console/apistatuslayout_test.go

@@ -181,8 +181,24 @@ func TestLayoutMixedNeedsANestedAndAFlatRow(t *testing.T) {

{[]string{"foreign", "external", "flat"}, false},

{nil, false},

} {

- if got := layoutMixed(rows(tc.layouts...)); got != tc.want {

- t.Errorf("layoutMixed(%v) = %v, want %v", tc.layouts, got, tc.want)

+ if got := layoutMixed("flat", rows(tc.layouts...)); got != tc.want {

+ t.Errorf("layoutMixed(flat, %v) = %v, want %v", tc.layouts, got, tc.want)

+ }

+ // Under a nested intent an unisolated row is mixed with or without a

+ // nested sibling; an all-nested or empty box is not.

+ for _, tc := range []struct {

+ layouts []string

+ want bool

+ }{

+ {[]string{"foreign", "external"}, true},

+ {[]string{"flat"}, true},

+ {[]string{"nested", "nested"}, false},

+ {[]string{"fenced"}, false},

+ {nil, false},

+ } {

+ if got := layoutMixed("nested", rows(tc.layouts...)); got != tc.want {

+ t.Errorf("layoutMixed(nested, %v) = %v, want %v", tc.layouts, got, tc.want)

}

diff --git a/internal/console/apisystemlayout.go b/internal/console/apisystemlayout.go

new file mode 100644

index 000000000..f1979728b

--- /dev/null

+++ b/internal/console/apisystemlayout.go

@@ -0,0 +1,117 @@

+package console

+

+import (

+ "net/http"

+

+ "github.com/carlosframework/platform/internal/controlname"

+ "github.com/carlosframework/platform/internal/release"

+)

+

+// The layout door (isolated-exec design §8.2, plan 5): `carlos system

+// layout migrate|revert|status`. POST records the fleet's intent in

+// control/layout.json and bumps the serial; every box in scope converges

+// on its next serial move. The console moves no file itself: the pass on

+// each box does, with the box's own registry as the authority on what is

+// eligible, so this door is a request, never an instruction about paths.

+//

+// GATE: apiSystemRoller, the same as the update and rollback doors. The

+// intent moves every tenant's files on the boxes in scope, which is the

+// blast radius of a platform update, so it is the same people: an owner of

+// the ops account or an address on CARLOS_SYSTEM_OPERATORS. Anyone else

+// gets the bare 404 those doors give. Registered through cliRoute with an

+// EMPTY role set: no service credential of any role reaches it.

+

+// apiSystemLayoutResponse is both doors' answer. Mode is "none" when no

+// request has ever been recorded; Box is "" for the fleet.

+type apiSystemLayoutResponse struct {

+ Mode string `json:"mode"`

+ Box string `json:"box"`

+ ID string `json:"id"`

+ RequestedAt string `json:"requested_at"`

+ Force bool `json:"force"`

+ // Requests is how many requests the log holds. Additive; the CLI

+ // ignores it today.

+ Requests int `json:"requests"`

+}

+

+func (s *Server) handleAPISystemLayoutGet(w http.ResponseWriter, r *http.Request) {

+ if _, _, ok := s.apiSystemRoller(w, r); !ok {

+ return

+ }

+ if !s.systemBucket(w) {

+ return

+ }

+ log, found, err := release.LoadLayoutIntents(r.Context(), s.cfg.Releases)

+ if err != nil {

+ jsonError(w, http.StatusBadGateway, "could not read the layout intent: "+err.Error())

+ return

+ }

+ out := apiSystemLayoutResponse{Mode: "none"}

+ if found && len(log.Requests) > 0 {

+ in := log.Requests[len(log.Requests)-1]

+ out = apiSystemLayoutResponse{Mode: in.Mode, Box: in.Scope.Box, ID: in.ID, RequestedAt: in.RequestedAt, Force: in.Force, Requests: len(log.Requests)}

+ }

+ writeJSON(w, http.StatusOK, out)

+}

+

+func (s *Server) handleAPISystemLayoutSet(w http.ResponseWriter, r *http.Request) {

+ addr, _, ok := s.apiSystemRoller(w, r)

+ if !ok {

+ return

+ }

+ if !s.systemBucket(w) {

+ return

+ }

+ var in struct {

+ Mode string `json:"mode"`

+ Box string `json:"box"`

+ Force bool `json:"force"`

+ }

+ if !readJSONStrictLimit(w, r, &in, 1<<10) {

+ return

+ }

+ if !release.ValidLayoutMode(in.Mode) {

+ jsonError(w, http.StatusBadRequest, "mode must be nested or flat")

+ return

+ }

+ // The label becomes a scope every box compares its own label against;

+ // the same shape the fleets doors accept for a box label.

+ if in.Box != "" && !controlname.ValidLabel(in.Box) {

+ jsonError(w, http.StatusBadRequest, "box must be a box label")

+ return

+ }

+ // A revert is refused while a maintenance worker is attached to a box

+ // in scope (#132's interlock): flat tenants would read the worker's

+ // credential and its private input through /proc the moment they ran

+ // flat again.

+ if in.Mode == release.LayoutModeFlat {

+ if reason := s.layoutRevertBlocked(r.Context(), in.Box); reason != "" {

+ jsonError(w, http.StatusConflict, reason)

+ return

+ }

+ // A migrate is refused while any box in scope runs below the floor

+ // release (layoutguard.go): a nested row there would never wake.

+ if in.Mode == release.LayoutModeNested {

+ if reason := s.layoutMigrateBlocked(r.Context(), in.Box); reason != "" {

+ jsonError(w, http.StatusConflict, reason)

+ return

+ }

+ rec, err := release.RecordLayoutIntent(r.Context(), s.cfg.Releases, in.Mode, in.Box, in.Force)

+ if err != nil {

+ jsonError(w, http.StatusBadGateway, err.Error())

+ return

+ }

+ scope := "the fleet"

+ if in.Box != "" {

+ scope = "box " + in.Box

+ }

+ detail := "recorded layout " + in.Mode + " for " + scope

+ if in.Force {

+ detail += " (force: awake instances are stopped for the move at once)"

+ }

+ s.auditSystem(addr, "system-layout", detail)

+ s.bumpSerial(r.Context())

+ writeJSON(w, http.StatusOK, apiSystemLayoutResponse{Mode: rec.Mode, Box: rec.Scope.Box, ID: rec.ID, RequestedAt: rec.RequestedAt, Force: rec.Force})

+}

diff --git a/internal/console/apisystemlayout_test.go b/internal/console/apisystemlayout_test.go

new file mode 100644

index 000000000..12cef7fd9

--- /dev/null

+++ b/internal/console/apisystemlayout_test.go

@@ -0,0 +1,160 @@

+package console

+

+import (

+ "context"

+ "errors"

+ "net/http"

+ "strings"

+ "testing"

+ "time"

+

+ "github.com/carlosframework/platform/internal/release"

+)

+

+// The layout door is the update door's gate, byte for byte: a member of

+// the ops account gets the bare 404 an unknown path gives, and a system

+// operator who is not an owner gets in, as they do at update and rollback.

+func TestSystemLayoutDoorIsTheSystemGate(t *testing.T) {

+ fx := newConsoleFixture(t)

+ if err := fx.st.AddMember(OpsAccountID, "member@keymail.dev", RoleMember, time.Now()); err != nil {

+ t.Fatal(err)

+ }

+ member := cliToken(t, fx, "member@keymail.dev", "code-layout-member")

+ _, want := getRaw(t, fx.ts.URL+"/api/cli/system/no-such-door", member)

+ if st, body := getRaw(t, fx.ts.URL+"/api/cli/system/layout", member); st != http.StatusNotFound || body != want {

+ t.Errorf("GET as a member = %d %q, want 404 %q", st, body, want)

+ }

+ if st, body := postRaw(t, fx.ts.URL+"/api/cli/system/layout", map[string]string{"mode": "nested"}, member); st != http.StatusNotFound || body != want {

+ t.Errorf("POST as a member = %d %q, want 404 %q", st, body, want)

+ }

+ // A member that nothing was recorded for: the object stays absent.

+ if _, found, _ := release.LoadLayoutIntents(context.Background(), fx.rel); found {

+ t.Error("a refused POST recorded an intent")

+ }

+

+ fx.srv.cfg.SystemOperators = []string{"Roller@keymail.dev"}

+ if err := fx.st.AddMember(OpsAccountID, "roller@keymail.dev", RoleMember, time.Now()); err != nil {

+ t.Fatal(err)

+ }

+ roller := cliToken(t, fx, "roller@keymail.dev", "code-layout-roller")

+ if st, out := getJSON(t, fx.ts.URL+"/api/cli/system/layout", roller); st != http.StatusOK || out["mode"] != "none" {

+ t.Errorf("GET as a system operator = %d %v, want 200 mode none", st, out)

+ }

+}

+

+func TestSystemLayoutDoorRecordsTheIntentAndBumpsTheSerial(t *testing.T) {

+ fx := newConsoleFixture(t)

+ owner := cliToken(t, fx, "paul@keymail.dev", "code-layout-owner")

+ // Every box in scope runs the floor release, so the migrate's floor

+ // check (TestSystemLayoutMigrateRefusedBelowTheFloor) is not what this

+ // test is about.

+ seedBox(t, fx, "kass-1", "linux-arm64", "stable", "v0.25.0")

+

+ for _, body := range []map[string]any{{"mode": "sideways"}, {"mode": ""}, {"mode": "nested", "box": "Not A Label!"}, {"mode": "nested", "bogus": 1}} {

+ if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/layout", body, owner); st != http.StatusBadRequest {

+ t.Errorf("POST %v = %d %v, want 400", body, st, out)

+ }

+ if serial(t, fx) != 0 {

+ t.Fatal("a refused request bumped the serial")

+ }

+

+ st, out := postJSON(t, fx.ts.URL+"/api/cli/system/layout", map[string]any{"mode": "nested", "box": "kass-1", "force": true}, owner)

+ if st != http.StatusOK || out["mode"] != "nested" || out["box"] != "kass-1" || out["force"] != true || str(t, out, "id") == "" {

+ t.Fatalf("record = %d %v", st, out)

+ }

+ in, found, err := release.LoadLayoutIntent(context.Background(), fx.rel, "kass-1")

+ if err != nil || !found || in.Mode != "nested" || in.Scope.Box != "kass-1" || !in.Force || in.ID != str(t, out, "id") {

+ t.Fatalf("object = %+v %v %v", in, found, err)

+ }

+ if _, found, _ := release.LoadLayoutIntent(context.Background(), fx.rel, "kass-2"); found {

+ t.Error("a box-scoped intent applies to another box")

+ }

+ if got := serial(t, fx); got != 1 {

+ t.Fatalf("serial = %d, want 1: boxes converge on the serial move", got)

+ }

+

+ st, out = getJSON(t, fx.ts.URL+"/api/cli/system/layout", owner)

+ if st != http.StatusOK || out["mode"] != "nested" || out["box"] != "kass-1" || out["force"] != true || out["requests"] != float64(1) {

+ t.Fatalf("get = %d %v", st, out)

+ }

+

+ // A fleet-wide revert after it is the newest request everywhere.

+ st, out = postJSON(t, fx.ts.URL+"/api/cli/system/layout", map[string]any{"mode": "flat"}, owner)

+ if st != http.StatusOK || out["box"] != "" || out["force"] != false {

+ t.Fatalf("revert = %d %v", st, out)

+ }

+ in, found, _ = release.LoadLayoutIntent(context.Background(), fx.rel, "kass-1")

+ if !found || in.Mode != "flat" {

+ t.Fatalf("kass-1 after a fleet-wide revert = %+v", in)

+ }

+ if got := serial(t, fx); got != 2 {

+ t.Fatalf("serial = %d, want 2", got)

+ }

+ // The audit row names the act.

+ events, err := fx.st.Events(OpsAccountID, "platform", 10)

+ if err != nil {

+ t.Fatal(err)

+ }

+ var saw bool

+ for _, e := range events {

+ if e.Action == "system-layout" && strings.Contains(e.Detail, "nested for box kass-1") && strings.Contains(e.Detail, "force") {

+ saw = true

+ }

+ if !saw {

+ t.Errorf("audit rows = %+v, want a system-layout row naming nested, box kass-1 and force", events)

+ }

+}

+

+func TestSystemLayoutDoorWithoutABucketRefuses(t *testing.T) {

+ fx := newConsoleFixture(t)

+ owner := cliToken(t, fx, "paul@keymail.dev", "code-layout-nobucket")

+ fx.srv.cfg.Releases = nil

+ if st, out := getJSON(t, fx.ts.URL+"/api/cli/system/layout", owner); st != http.StatusServiceUnavailable {

+ t.Errorf("GET without a bucket = %d %v, want 503", st, out)

+ }

+ if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/layout", map[string]string{"mode": "nested"}, owner); st != http.StatusServiceUnavailable {

+ t.Errorf("POST without a bucket = %d %v, want 503", st, out)

+ }

+}

+

+// #132's interlock: a revert is refused for a box with an active or

+// revoking maintenance worker head, in scope. A detached head, or a head

+// on a box outside a box-scoped revert, does not block. Migrating is never

+// blocked by a worker. A reader that fails blocks: unknown is not "none".

+func TestSystemLayoutRevertRefusedWhileAWorkerHeadIsAttached(t *testing.T) {

+ fx := newConsoleFixture(t)

+ owner := cliToken(t, fx, "paul@keymail.dev", "code-layout-worker")

+ heads := []release.WorkerHead{

+ {Box: "kass-1", Status: "active"},

+ {Box: "kass-2", Status: "detached"},

+ {Box: "kass-3", Status: "revoking"},

+ }

+ var readErr error

+ fx.srv.cfg.WorkerHeads = func(context.Context) ([]release.WorkerHead, error) { return heads, readErr }

+ for _, b := range []string{"kass-1", "kass-2", "kass-3"} {

+ seedBox(t, fx, b, "linux-arm64", "stable", "v0.25.0")

+ }

+

+ st, out := postJSON(t, fx.ts.URL+"/api/cli/system/layout", map[string]string{"mode": "flat"}, owner)

+ if st != http.StatusConflict {

+ t.Fatalf("fleet revert with workers attached = %d %v, want 409", st, out)

+ }

+ if msg := str(t, out, "error"); !strings.Contains(msg, "kass-1 has a maintenance worker (active)") || !strings.Contains(msg, "kass-3 has a maintenance worker (revoking)") || strings.Contains(msg, "kass-2") {

+ t.Errorf("reason = %q, want kass-1 and kass-3 named and kass-2 not", msg)

+ }

+ if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/layout", map[string]string{"mode": "flat", "box": "kass-1"}, owner); st != http.StatusConflict {

+ t.Errorf("box revert on the worker's box = %d %v, want 409", st, out)

+ }

+ if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/layout", map[string]string{"mode": "flat", "box": "kass-2"}, owner); st != http.StatusOK {

+ t.Errorf("box revert beside a detached head = %d %v, want 200", st, out)

+ }

+ if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/layout", map[string]string{"mode": "nested"}, owner); st != http.StatusOK {

+ t.Errorf("migrate with workers attached = %d %v, want 200", st, out)

+ }

+ readErr = errors.New("store down")

+ if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/layout", map[string]string{"mode": "flat", "box": "kass-2"}, owner); st != http.StatusConflict {

+ t.Errorf("revert with an unreadable head store = %d %v, want 409", st, out)

+ }

+}

diff --git a/internal/console/apisystemupdate.go b/internal/console/apisystemupdate.go

index 9640e7655..3f56ad84d 100644

--- a/internal/console/apisystemupdate.go

+++ b/internal/console/apisystemupdate.go

@@ -334,6 +334,17 @@ func (s *Server) handleAPISystemUpdate(w http.ResponseWriter, r *http.Request) {

}

spread = d

}

+ // A named release below the layout floor is refused here, before a

+ // job starts, while any box reports a nested, fenced or conflict row

+ // or no layout at all (layoutguard.go). Only a named version can be

+ // judged: "the latest public release" is resolved inside the job and

+ // is never below the floor, since the floor is a past release.

+ if in.Version != "" && in.Upload == "" {

+ if reason := s.layoutDowngradeBlocked(r.Context(), in.Version); reason != "" {

+ jsonError(w, http.StatusConflict, reason)

+ return

+ }

id, job, busy := s.startSystemJob(in.Component, in.Channel)

if busy != "" {

writeJSON(w, http.StatusConflict, map[string]string{

@@ -704,6 +715,17 @@ func (s *Server) handleAPISystemRollback(w http.ResponseWriter, r *http.Request)

jsonError(w, http.StatusBadRequest, "channel must be canary or stable")

return

}

+ // The rollback target is the pointer's History[0], exactly what

+ // RollbackSystem will move to; it is read first so the layout guard

+ // can judge it before the pointer moves (design §8.2, "The console's

+ // rollback door"). A pointer that cannot be read here is left for

+ // RollbackSystem to report in its own words below.

+ if cur, _, err := release.GetSystemPointer(r.Context(), s.cfg.Releases, s.cfg.ReleasePub, in.Component, in.Channel); err == nil && len(cur.History) > 0 {

+ if reason := s.layoutDowngradeBlocked(r.Context(), cur.History[0].Version); reason != "" {

+ jsonError(w, http.StatusConflict, reason)

+ return

+ }

p, err := release.RollbackSystem(r.Context(), s.cfg.Releases, release.RootSigner(s.cfg.Key), in.Component, in.Channel, addr)

switch {

case errors.Is(err, release.ErrNoSystemHistory):

diff --git a/internal/console/apisystemupdate_test.go b/internal/console/apisystemupdate_test.go

index 63c396000..9a7ddffe1 100644

--- a/internal/console/apisystemupdate_test.go

+++ b/internal/console/apisystemupdate_test.go

@@ -425,17 +425,21 @@ func TestSystemUpdateFromAnUpload(t *testing.T) {

}

+// Versions here are at or above release.LayoutFloor: since plan 5 the

+// rollback door judges the release it would move back to, and a version

+// that cannot be placed against the floor is refused while any box reports

+// no layout (layoutguard_test.go covers that side).

func TestSystemStatusAndRollback(t *testing.T) {

fx := newConsoleFixture(t)

owner := cliToken(t, fx, "paul@keymail.dev", "code-sys-status")

- seedBox(t, fx, "ie", "linux-arm64", "canary", "v1")

+ seedBox(t, fx, "ie", "linux-arm64", "canary", "v0.25.0")

sg := release.RootSigner(fx.priv)

rel := func(v string) release.SystemRelease {

return release.SystemRelease{Version: v, Artifacts: map[string]release.SystemArtifact{"linux-arm64": {Key: release.SystemArtifactKey(v, "linux-arm64"), SHA256: strings.Repeat("a", 64)}}}

}

- release.PromoteSystem(context.Background(), fx.rel, sg, "router", "canary", rel("v1"), release.SystemPromoteOptions{})

+ release.PromoteSystem(context.Background(), fx.rel, sg, "router", "canary", rel("v0.25.0"), release.SystemPromoteOptions{})

time.Sleep(1100 * time.Millisecond)

- release.PromoteSystem(context.Background(), fx.rel, sg, "router", "canary", rel("v2"), release.SystemPromoteOptions{})

+ release.PromoteSystem(context.Background(), fx.rel, sg, "router", "canary", rel("v0.26.0"), release.SystemPromoteOptions{})

st, out := getJSON(t, fx.ts.URL+"/api/cli/system/status", owner)

if st != http.StatusOK {

t.Fatalf("status = %d %v", st, out)

@@ -445,12 +449,12 @@ func TestSystemStatusAndRollback(t *testing.T) {

t.Fatalf("boxes = %v", boxes)

}

b := boxes[0].(map[string]any)

- if b["label"] != "ie" || b["release"] != "v1" || b["target"] != "v2" || b["channel"] != "canary" {

+ if b["label"] != "ie" || b["release"] != "v0.25.0" || b["target"] != "v0.26.0" || b["channel"] != "canary" {

t.Fatalf("box row = %v", b)

}

time.Sleep(1100 * time.Millisecond)

st, out = postJSON(t, fx.ts.URL+"/api/cli/system/rollback", map[string]string{"channel": "canary"}, owner)

- if st != http.StatusOK || str(t, out, "version") != "v1" {

+ if st != http.StatusOK || str(t, out, "version") != "v0.25.0" {

t.Fatalf("rollback = %d %v", st, out)

}

if st, out = postJSON(t, fx.ts.URL+"/api/cli/system/rollback", map[string]string{"channel": "canary"}, owner); st != http.StatusConflict {

diff --git a/internal/console/authcli.go b/internal/console/authcli.go

index 9bd9e0a22..184ec861c 100644

--- a/internal/console/authcli.go

+++ b/internal/console/authcli.go

@@ -879,6 +879,8 @@ func (s *Server) registerCLIRoutes() {

s.cliRoute("GET /api/cli/system/jobs/{id}", svcrole.Set{}, s.handleAPISystemJob)

s.cliRoute("PUT /api/cli/system/upload", svcrole.Set{}, s.handleAPISystemUpload)

s.cliRoute("POST /api/cli/system/rollback", svcrole.Set{}, s.handleAPISystemRollback)

+ s.cliRoute("GET /api/cli/system/layout", svcrole.Set{}, s.handleAPISystemLayoutGet)

+ s.cliRoute("POST /api/cli/system/layout", svcrole.Set{}, s.handleAPISystemLayoutSet)

s.cliRoute("GET /api/cli/system/antivirus", svcrole.Set{}, s.handleAPISystemAntivirusGet)

s.cliRoute("POST /api/cli/system/antivirus", svcrole.Set{}, s.handleAPISystemAntivirusSet)

}

diff --git a/internal/console/layoutguard.go b/internal/console/layoutguard.go

new file mode 100644

index 000000000..a2f3ea488

--- /dev/null

+++ b/internal/console/layoutguard.go

@@ -0,0 +1,154 @@

+package console

+

+import (

+ "context"

+ "sort"

+ "strings"

+ "time"

+

+ "github.com/carlosframework/platform/internal/release"

+ "github.com/carlosframework/platform/internal/status"

+)

+

+// layoutDowngradeBlocked is design §8.2's rollback rule at the console: a

+// channel pointer must not move below release.LayoutFloor while any box

+// could still hold a nested row. The box-side guard (host/systemapply.go)

+// refuses the install too, but only on the box that runs it; this one

+// stops the pointer moving at all, so no box has to refuse.

+//

+// It judges OBSERVED state, never the intent: a revert refused on a box

+// leaves a nested row under a flat intent. A box that has not reported in

+// time, or whose snapshot carries no layout at all, is unknown and blocks,

+// because the cost of guessing is a database hidden under a mount. The

+// empty string means the move is allowed.

+func (s *Server) layoutDowngradeBlocked(ctx context.Context, target string) string {

+ if !release.PreLayout(target) {

+ return ""

+ }

+ // Every fleet's boxes, the union carlos system status reads. Strict:

+ // a fleet whose heartbeats cannot be read is a fleet whose rows are

+ // unknown, and unknown blocks.

+ boxes, _, err := s.systemBoxes(ctx, true)

+ if err != nil {

+ return target + " predates the instance layout and the fleet's layout is unknown: could not read the boxes' snapshots: " + err.Error()

+ }

+ now := time.Now()

+ var reasons []string

+ for _, b := range boxes {

+ switch {

+ case status.BoxStale(b, now, status.SnapshotPeriod):

+ reasons = append(reasons, b.Label+" is stale")

+ case b.Metrics == nil || b.Metrics.Layout == nil:

+ // A box whose running release parses and is below the floor

+ // cannot have run the layout pass (the pass and its report

+ // arrived together, in the floor release), so its silence is

+ // flat: today's fleet, on v0.21.0 to v0.24.2, keeps its

+ // rollback. Silence at or above the floor, or from a box whose

+ // release is a sha or "dev" (which could be a build with the

+ // pass), is unknown and blocks. A box at or above the floor

+ // with no activator publishes a report of its own (Mode

+ // "unmanaged", host/layoutleg.go), so it is not silent here.

+ if b.Metrics != nil && release.BelowFloor(b.Metrics.Release, release.LayoutFloor) {

+ continue

+ }

+ reasons = append(reasons, b.Label+" reports no layout field")

+ default:

+ for host, row := range b.Metrics.Layout.Rows {

+ switch row.Layout {

+ case "nested", "fenced", "conflict":

+ reasons = append(reasons, b.Label+": "+host+" is "+row.Layout)

+ }

+ if len(reasons) == 0 {

+ return ""

+ }

+ sort.Strings(reasons)

+ return target + " predates the instance layout and the fleet is not flat: " + strings.Join(reasons, "; ") +

+ ". Run `carlos system layout revert`, wait for `carlos status --layout` to show every row flat, then retry"

+}

+

+// layoutMigrateBlocked refuses `layout migrate` while any box in scope

+// (every box for "", else the one labelled box) runs a release below

+// release.LayoutFloor, runs one this console cannot place and reports no

+// layout, or is stale: a nested row minted or moved on such a box would never

+// wake, so nothing may migrate before the floor release is cut and rolled.

+// A named box that no snapshot knows is refused too: an intent for a box

+// nobody can see is one nobody can check.

+func (s *Server) layoutMigrateBlocked(ctx context.Context, box string) string {

+ boxes, _, err := s.systemBoxes(ctx, true)

+ if err != nil {

+ return "layout migrate refused: could not read the boxes' snapshots: " + err.Error()

+ }

+ now := time.Now()

+ var reasons []string

+ seen := false

+ for _, b := range boxes {

+ if box != "" && b.Label != box {

+ continue

+ }

+ seen = true

+ switch {

+ case status.BoxStale(b, now, status.SnapshotPeriod):

+ reasons = append(reasons, b.Label+" is stale")

+ case b.Metrics == nil || b.Metrics.Release == "":

+ reasons = append(reasons, b.Label+" reports no release")

+ case release.BelowFloor(b.Metrics.Release, release.LayoutFloor):

+ reasons = append(reasons, b.Label+" runs "+b.Metrics.Release)

+ case release.PreLayout(b.Metrics.Release) && b.Metrics.Layout == nil:

+ // A sha or "dev" (a dev box tracking main) cannot be placed

+ // against the floor, but a layout report proves the build

+ // carries the pass, which is what the floor stands for; with

+ // no report there is no such proof.

+ reasons = append(reasons, b.Label+" runs "+b.Metrics.Release+", which cannot be placed against v"+release.LayoutFloor.String()+", and reports no layout")

+ }

+ if box != "" && !seen {

+ reasons = append(reasons, "box "+box+" has not reported")

+ }

+ if len(reasons) == 0 {

+ return ""

+ }

+ sort.Strings(reasons)

+ return "layout migrate refused: every box in scope must run v" + release.LayoutFloor.String() + " or later and be reporting: " + strings.Join(reasons, "; ")

+}

+

+// layoutRevertBlocked is #132's interlock on the layout door: a revert is

+// refused for any box in scope ("" is every box) that has a maintenance

+// worker head whose status is active or revoking. A worker's bearer

+// credential and its private input live on the box as the agent's uid,

+// and a flat tenant shares that uid and the agent's /proc view, so running

+// a tenant flat beside a worker hands it both. A detached head holds no

+// credential and does not block.

+//

+// Heads are read through cfg.WorkerHeads. On main the signed WorkerHead

+// shape exists (release/workercontrol.go) but no object key, store reader

+// or lifecycle writer does yet: #132's codec and store slice landed with

+// "no runtime caller". TODO(#132): wire WorkerHeads to the head store once

+// its key and reader land, and have the box-side revert pass refuse while

+// a durable worker token exists, as #132 also asks. Until then no worker

+// is enrolled anywhere (#132, #126), and a nil seam reads as "no heads".

+func (s *Server) layoutRevertBlocked(ctx context.Context, box string) string {

+ if s.cfg.WorkerHeads == nil {

+ return ""

+ }

+ heads, err := s.cfg.WorkerHeads(ctx)

+ if err != nil {

+ return "could not read the maintenance worker heads, so the revert is refused: " + err.Error()

+ }

+ var reasons []string

+ for _, h := range heads {

+ if box != "" && h.Box != box {

+ continue

+ }

+ if h.Status == "active" || h.Status == "revoking" {

+ reasons = append(reasons, h.Box+" has a maintenance worker ("+h.Status+")")

+ }

+ if len(reasons) == 0 {

+ return ""

+ }

+ sort.Strings(reasons)

+ return "layout revert refused: " + strings.Join(reasons, "; ") + ". A flat tenant shares the worker's user and can read its credential; detach the worker first"

+}

diff --git a/internal/console/layoutguard_test.go b/internal/console/layoutguard_test.go

new file mode 100644

index 000000000..57467f163

--- /dev/null

+++ b/internal/console/layoutguard_test.go

@@ -0,0 +1,215 @@

+package console

+

+import (

+ "context"

+ "fmt"

+ "net/http"

+ "strings"

+ "testing"

+ "time"

+

+ "github.com/carlosframework/platform/internal/release"

+ "github.com/carlosframework/platform/internal/status"

+)

+

+// layoutDowngradeBlocked judges observed state: a target at or above the

+// floor is always allowed; below it, every box must be fresh, carry a

+// layout report, and report no nested, fenced or conflict row.

+func TestLayoutDowngradeBlockedJudgesEveryBoxsReport(t *testing.T) {

+ fx := newConsoleFixture(t)

+ ctx := context.Background()

+ now := time.Now()

+ flat := &status.LayoutReport{Mode: "flat", Rows: map[string]status.RowLayout{"a.example.com": {Layout: "flat"}, "x.example.com": {Layout: "external"}}}

+

+ seedLayoutBox(t, fx.rel, "ie", now, flat)

+ if got := fx.srv.layoutDowngradeBlocked(ctx, "v0.19.0"); got != "" {

+ t.Fatalf("all flat: %q, want allowed", got)

+ }

+

+ seedLayoutBox(t, fx.rel, "uk", now, &status.LayoutReport{Mode: "nested", Rows: map[string]status.RowLayout{

+ "b.example.com": {Layout: "nested"}, "c.example.com": {Layout: "flat"}, "d.example.com": {Layout: "fenced", Reason: "mid-move"},

+ }})

+ got := fx.srv.layoutDowngradeBlocked(ctx, "v0.19.0")

+ for _, want := range []string{"v0.19.0 predates the instance layout", "uk: b.example.com is nested", "uk: d.example.com is fenced", "carlos system layout revert"} {

+ if !strings.Contains(got, want) {

+ t.Errorf("reason %q lacks %q", got, want)

+ }

+ if strings.Contains(got, "c.example.com") || strings.Contains(got, "ie") {

+ t.Errorf("reason %q names a flat row or a flat box", got)

+ }

+ // At or above the floor nothing is judged.

+ for _, v := range []string{"v0.25.0", "v0.26.1"} {

+ if got := fx.srv.layoutDowngradeBlocked(ctx, v); got != "" {

+ t.Errorf("%s: %q, want allowed", v, got)

+ }

+

+ // A box that reports no layout at all is unknown and blocks, unless

+ // the release it reports running parses below the floor: that build

+ // has no layout pass, so its silence is flat. Today's fleet, on

+ // v0.21.0 to v0.24.2, keeps its rollback (and its named update).

+ for _, v := range []string{"v0.18.0", "v0.20.0", "v0.21.0", "v0.24.2"} {

+ seedBox(t, fx, "au", "linux-arm64", "stable", v)

+ if got := fx.srv.layoutDowngradeBlocked(ctx, "v0.19.0"); strings.Contains(got, "au ") {

+ t.Errorf("a %s box with no layout field blocked: %q", v, got)

+ }

+ // Silence at or above the floor is unknown: that build reports a

+ // layout when it has an activator, and says "unmanaged" when it has

+ // none, so no report at all means a box the console cannot vouch for.

+ seedBox(t, fx, "au", "linux-arm64", "stable", "v0.25.0")

+ if got := fx.srv.layoutDowngradeBlocked(ctx, "v0.19.0"); !strings.Contains(got, "au reports no layout field") {

+ t.Errorf("no layout field on the floor release: %q", got)

+ }

+ seedBox(t, fx, "au", "linux-arm64", "stable", "dev")

+ if got := fx.srv.layoutDowngradeBlocked(ctx, "v0.19.0"); !strings.Contains(got, "au reports no layout field") {

+ t.Errorf("no layout field on an unplaceable release: %q", got)

+ }

+ // So does a stale one, whatever its last report said.

+ seedLayoutBox(t, fx.rel, "au", now.Add(-20*time.Minute), flat)

+ if got := fx.srv.layoutDowngradeBlocked(ctx, "v0.19.0"); !strings.Contains(got, "au is stale") {

+ t.Errorf("stale: %q", got)

+ }

+}

+

+// Both system doors call the guard before a pointer moves: the rollback

+// door against the release the pointer would go back to, the update door

+// against a named version. A non-nested fleet rolls back as before.

+func TestSystemDoorsRefuseAPreLayoutMoveWhileABoxIsNested(t *testing.T) {

+ fx := newConsoleFixture(t)

+ owner := cliToken(t, fx, "paul@keymail.dev", "code-layout-guard")

+ ctx := context.Background()

+ sg := release.RootSigner(fx.priv)

+ rel := func(v string) release.SystemRelease {

+ return release.SystemRelease{Version: v, Artifacts: map[string]release.SystemArtifact{"linux-arm64": {Key: release.SystemArtifactKey(v, "linux-arm64"), SHA256: strings.Repeat("a", 64)}}}

+ }

+ if _, err := release.PromoteSystem(ctx, fx.rel, sg, "router", "canary", rel("v0.19.0"), release.SystemPromoteOptions{}); err != nil {

+ t.Fatal(err)

+ }

+ time.Sleep(1100 * time.Millisecond)

+ if _, err := release.PromoteSystem(ctx, fx.rel, sg, "router", "canary", rel("v0.25.0"), release.SystemPromoteOptions{}); err != nil {

+ t.Fatal(err)

+ }

+ seedLayoutBox(t, fx.rel, "ie", time.Now(), &status.LayoutReport{Mode: "nested", Rows: map[string]status.RowLayout{"a.example.com": {Layout: "nested"}}})

+

+ st, out := postJSON(t, fx.ts.URL+"/api/cli/system/rollback", map[string]string{"channel": "canary"}, owner)

+ if st != http.StatusConflict || !strings.Contains(str(t, out, "error"), "ie: a.example.com is nested") {

+ t.Fatalf("rollback to v0.19.0 with a nested row = %d %v, want 409 naming the row", st, out)

+ }

+ p, _, err := release.GetSystemPointer(ctx, fx.rel, fx.pub, "router", "canary")

+ if err != nil || p.Current.Version != "v0.25.0" {

+ t.Fatalf("pointer moved anyway: %+v %v", p, err)

+ }

+ st, out = postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "canary", "version": "v0.19.0"}, owner)

+ if st != http.StatusConflict {

+ t.Fatalf("update to v0.19.0 with a nested row = %d %v, want 409", st, out)

+ }

+ // A version at the floor or above is not the guard's business; the

+ // request goes on to the job (and fails there, on this fixture's

+ // absent feed, which is not what this test is about).

+ if st, _ := postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "canary", "version": "v0.26.0"}, owner); st == http.StatusConflict {

+ t.Error("update to v0.26.0 was refused by the layout guard")

+ }

+

+ // Every row flat: the rollback goes through.

+ seedLayoutBox(t, fx.rel, "ie", time.Now(), &status.LayoutReport{Mode: "flat", Rows: map[string]status.RowLayout{"a.example.com": {Layout: "flat"}}})

+ time.Sleep(1100 * time.Millisecond)

+ if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/rollback", map[string]string{"channel": "canary"}, owner); st != http.StatusOK || str(t, out, "version") != "v0.19.0" {

+ t.Fatalf("rollback with every row flat = %d %v", st, out)

+ }

+}

+

+// `layout migrate` is refused while any box in scope runs below the floor,

+// is stale, reports no release, or (for a named box) has never reported;

+// it goes through once every box in scope runs the floor release. A

+// revert is never judged this way.

+func TestSystemLayoutMigrateRefusedBelowTheFloor(t *testing.T) {

+ fx := newConsoleFixture(t)

+ owner := cliToken(t, fx, "paul@keymail.dev", "code-layout-floor")

+ seedBox(t, fx, "ie", "linux-arm64", "stable", "v0.24.2")

+ seedBox(t, fx, "uk", "linux-arm64", "stable", "v0.25.0")

+ st, out := postJSON(t, fx.ts.URL+"/api/cli/system/layout", map[string]string{"mode": "nested"}, owner)

+ if st != http.StatusConflict || !strings.Contains(str(t, out, "error"), "ie runs v0.24.2") || strings.Contains(str(t, out, "error"), "uk") {

+ t.Fatalf("fleet migrate with a box below the floor = %d %v, want 409 naming ie only", st, out)

+ }

+ if _, found, _ := release.LoadLayoutIntents(context.Background(), fx.rel); found {

+ t.Fatal("a refused migrate recorded an intent")

+ }

+ if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/layout", map[string]string{"mode": "nested", "box": "ie"}, owner); st != http.StatusConflict {

+ t.Errorf("box migrate on the low box = %d %v, want 409", st, out)

+ }

+ if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/layout", map[string]string{"mode": "nested", "box": "au"}, owner); st != http.StatusConflict || !strings.Contains(str(t, out, "error"), "box au has not reported") {

+ t.Errorf("box migrate on an unknown box = %d %v, want 409 saying it has not reported", st, out)

+ }

+ if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/layout", map[string]string{"mode": "nested", "box": "uk"}, owner); st != http.StatusOK {

+ t.Errorf("box migrate on the floor-release box = %d %v, want 200", st, out)

+ }

+ // A revert is not judged by the floor: the low box can always go flat.

+ if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/layout", map[string]string{"mode": "flat", "box": "ie"}, owner); st != http.StatusOK {

+ t.Errorf("revert on the low box = %d %v, want 200", st, out)

+ }

+ seedBox(t, fx, "ie", "linux-arm64", "stable", "v0.25.0")

+ if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/layout", map[string]string{"mode": "nested"}, owner); st != http.StatusOK {

+ t.Errorf("fleet migrate with every box on the floor release = %d %v, want 200", st, out)

+ }

+ // An unplaceable release (a dev box on a sha) blocks while it reports

+ // no layout, and passes once its layout report is there: the report

+ // is proof the build carries the pass.

+ seedBox(t, fx, "ie", "linux-arm64", "stable", "abc1234")

+ if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/layout", map[string]string{"mode": "nested"}, owner); st != http.StatusConflict || !strings.Contains(str(t, out, "error"), "ie runs abc1234, which cannot be placed against v0.25.0, and reports no layout") {

+ t.Errorf("fleet migrate with an unplaceable release and no layout = %d %v, want 409", st, out)

+ }

+ b := status.CollectBox(nil, status.Probes{Release: "abc1234", Platform: "linux-arm64", SystemChannel: "stable",

+ Layout: &status.LayoutReport{Mode: "none", Rows: map[string]status.RowLayout{"a.example.com": {Layout: "flat"}}}}, time.Now())

+ b.Label = "ie"

+ if err := status.WriteBox(context.Background(), fx.rel, b); err != nil {

+ t.Fatal(err)

+ }

+ if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/layout", map[string]string{"mode": "nested"}, owner); st != http.StatusOK {

+ t.Errorf("fleet migrate with a sha box that reports its layout = %d %v, want 200", st, out)

+ }

+}

+

+// Today's fleet: every box on v0.21.0 to v0.24.2, none reporting a layout

+// field (those builds have no pass). Rollback and a named update to a

+// pre-floor release both go through, because nothing on such a box can be

+// nested. A box on a sha is a build that might carry the pass, so its

+// silence still blocks.

+func TestTodaysFleetWithNoLayoutFieldKeepsRollbackAndUpdate(t *testing.T) {

+ fx := newConsoleFixture(t)

+ owner := cliToken(t, fx, "paul@keymail.dev", "code-layout-todays-fleet")

+ ctx := context.Background()

+ for i, v := range []string{"v0.21.0", "v0.22.2", "v0.24.2", "v0.24.2"} {

+ seedBox(t, fx, fmt.Sprintf("box-%d", i), "linux-arm64", "stable", v)

+ }

+ sg := release.RootSigner(fx.priv)

+ rel := func(v string) release.SystemRelease {

+ return release.SystemRelease{Version: v, Artifacts: map[string]release.SystemArtifact{"linux-arm64": {Key: release.SystemArtifactKey(v, "linux-arm64"), SHA256: strings.Repeat("a", 64)}}}

+ }

+ if _, err := release.PromoteSystem(ctx, fx.rel, sg, "router", "stable", rel("v0.23.0"), release.SystemPromoteOptions{}); err != nil {

+ t.Fatal(err)

+ }

+ time.Sleep(1100 * time.Millisecond)

+ if _, err := release.PromoteSystem(ctx, fx.rel, sg, "router", "stable", rel("v0.24.2"), release.SystemPromoteOptions{}); err != nil {

+ t.Fatal(err)

+ }

+ if got := fx.srv.layoutDowngradeBlocked(ctx, "v0.23.0"); got != "" {

+ t.Fatalf("today's fleet blocked: %q", got)

+ }

+ time.Sleep(1100 * time.Millisecond)

+ if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/rollback", map[string]string{"channel": "stable"}, owner); st != http.StatusOK || str(t, out, "version") != "v0.23.0" {

+ t.Fatalf("rollback on today's fleet = %d %v, want 200 onto v0.23.0", st, out)

+ }

+ // A named pre-floor update passes the guard (and goes on to the job,

+ // which fails on this fixture's absent feed; not what this is about).

+ if st, out := postJSON(t, fx.ts.URL+"/api/cli/system/update", map[string]any{"channel": "stable", "version": "v0.24.2"}, owner); st == http.StatusConflict {

+ t.Fatalf("named update on today's fleet refused by the layout guard: %v", out)

+ }

+ // A box whose release is a sha is a build that might carry the pass,

+ // so its silence blocks.

+ seedBox(t, fx, "box-9", "linux-arm64", "stable", "abc1234")

+ if got := fx.srv.layoutDowngradeBlocked(ctx, "v0.23.0"); !strings.Contains(got, "box-9 reports no layout field") {

+ t.Fatalf("a sha box with no layout field: %q", got)

+ }

+}

diff --git a/internal/console/server.go b/internal/console/server.go

index 5eaefcbdd..d38acd44f 100644

--- a/internal/console/server.go

+++ b/internal/console/server.go

@@ -731,6 +731,11 @@ type Config struct {

// only an owner can put an unreleased build on it. Empty is the

// owner-only console that existed before the list.

SystemOperators []string

+ // WorkerHeads lists the deployment's maintenance worker heads (#132)

+ // for the layout door's revert interlock (layoutRevertBlocked). Nil

+ // means no reader is wired, which is every console today: the head

+ // store has no key or reader on main yet.

+ WorkerHeads func(ctx context.Context) ([]release.WorkerHead, error)

// PlatformDomain is the domain relay hostnames are built under

// (CARLOS_PLATFORM_DOMAIN; flagship: "oncarlos.com"), so an enabled

// app is handed stun:relay.<region>.<PlatformDomain>:3478 and its two

diff --git a/internal/console/svcroutes_test.go b/internal/console/svcroutes_test.go

index b9f7ecf30..b904f23de 100644

--- a/internal/console/svcroutes_test.go

+++ b/internal/console/svcroutes_test.go

@@ -206,6 +206,10 @@ var closedDoors = []string{

"GET /api/cli/system/jobs/{id}",

"PUT /api/cli/system/upload",

"POST /api/cli/system/rollback",

+ // The layout door (plan 5): the fleet's instance layout is the

+ // substrate every tenant sits inside, the same cut as the five above.

+ "GET /api/cli/system/layout",

+ "POST /api/cli/system/layout",

}

// TestEveryCLIDoorNamesItsRoles is the default-deny guarantee for the

@@ -819,8 +823,8 @@ func TestEveryClosedDoorRefusesAServiceCredential(t *testing.T) {

}

seen++

}

- if seen != len(closedDoors) || seen != 37 {

- t.Errorf("walked %d closed doors, want all %d (and 37 today: instance-token, three door-key doors, the routes door, instances-manifest, sidecars-manifest, the route-channel move, two db replica doors, six fleets, four services, two email credentials, email pause, the build probe, the live tenancy read, the layout read, five system doors, two deployment-wide antivirus, and four store-entitlement doors)",

+ if seen != len(closedDoors) || seen != 39 {

+ t.Errorf("walked %d closed doors, want all %d (and 39 today: instance-token, three door-key doors, the routes door, instances-manifest, sidecars-manifest, the route-channel move, two db replica doors, six fleets, four services, two email credentials, email pause, the build probe, the live tenancy read, the layout read, seven system doors, two deployment-wide antivirus, and four store-entitlement doors)",

seen, len(closedDoors))

}

diff --git a/internal/host/agent.go b/internal/host/agent.go

index 94dad4ed4..ac350b5b4 100644

--- a/internal/host/agent.go

+++ b/internal/host/agent.go

@@ -1033,6 +1033,14 @@ var reconcileMu sync.Mutex

func reconcileSerialized(ctx context.Context, s release.Store, reg *registry.Registry, cfg provision.ReconcileConfig) error {

reconcileMu.Lock()

defer reconcileMu.Unlock()

+ // A fresh row is minted at the layout this box's intent names

+ // (ReconcileConfig.Layout): nested on a nested box, so a new tenant

+ // never runs flat there. Set here, the one place both callers pass

+ // through, because the intent is read from the store this pass has in

+ // hand and scoped to the label this box publishes under.

+ if cfg.Layout == nil {

+ cfg.Layout = mintLayoutIntent

+ }

return errors.Join(

provision.Reconcile(ctx, s, reg, cfg),

provision.ReconcileSidecars(ctx, s, reg, cfg),

@@ -2993,10 +3001,11 @@ func statusLoop(reg *registry.Registry) {

// One stat (two with a WAL) per database route per tick, which

// is what `carlos instances list` and `carlos routes` show as DB.

DBBytes: dbFileBytes,

- // Layout is nil until this box has armed a layout pass (an

- // activator running, past startup recovery) — see

- // layoutStatusReport's own doc comment for why nil is the

- // right absence to publish rather than a guessed mode.

+ // Layout is the armed pass's classification of every

+ // instance-kind row, or, with no activator on this box, the

+ // rows judged by path shape under Mode "unmanaged" — see

+ // layoutStatusReport's own doc comment for why a box with no

+ // pass still says something rather than nothing.

Layout: layoutStatusReport(routes),

}

// The hosts this box is refusing because it cannot provide their

diff --git a/internal/host/agent_layout_test.go b/internal/host/agent_layout_test.go

index db82d66a7..5ab3b268a 100644

--- a/internal/host/agent_layout_test.go

+++ b/internal/host/agent_layout_test.go

@@ -500,10 +500,83 @@ func TestSyncExecSerializedSkipsAHostWithALayoutJournal(t *testing.T) {

// omission rule (R15): with no activator, and so no layout pass, there is

// nothing to move and nothing to report — the status snapshot must omit the

// block entirely rather than guess at a mode.

-func TestLayoutStatusReportWithNoPassArmedIsNil(t *testing.T) {

+// With no pass armed (no activator on the box) the report is not nil but

+// "unmanaged": rows judged by path shape, so the console's downgrade guard

+// can still tell a nested database from a flat one, and a box that never

+// arms an activator does not block its fleet's rollback by silence.

+func TestLayoutStatusReportWithNoPassArmedIsUnmanagedByPathShape(t *testing.T) {

resetLayoutGlobals(t)

- if got := layoutStatusReport(nil); got != nil {

- t.Fatalf("report = %+v, want nil with no pass armed", got)

+ t.Setenv("CARLOS_DATA_DIR", "/data")

+ got := layoutStatusReport([]registry.Route{

+ {Host: "nested.example.com", Kind: "instance", Hibernate: true, DBPath: provision.NestedDBPath("/data", "nested.example.com"), Socket: provision.NestedSocket("nested.example.com")},

+ // Unflagged but under the instance tree: still nested by shape.

+ {Host: "unflagged.example.com", Kind: "instance", DBPath: provision.NestedDBPath("/data", "unflagged.example.com"), Socket: provision.NestedSocket("unflagged.example.com")},

+ {Host: "flat.example.com", Kind: "instance", Hibernate: true, DBPath: provision.DefaultDBPath("/data", "flat.example.com"), Socket: provision.DefaultSocket("flat.example.com")},

+ {Host: "external.example.com", Kind: "instance", Socket: provision.DefaultSocket("external.example.com")},

+ {Host: "side.example.com", Kind: "sidecar"},

+ {Host: "static.example.com", Kind: "static"},

+ })

+ if got == nil || got.Mode != LayoutModeUnmanaged {

+ t.Fatalf("report = %+v, want Mode unmanaged with no pass armed", got)

+ }

+ want := map[string]string{"nested.example.com": "nested", "unflagged.example.com": "nested", "flat.example.com": "flat", "external.example.com": "external"}

+ if len(got.Rows) != len(want) {

+ t.Fatalf("rows = %+v, want %v", got.Rows, want)

+ }

+ for host, layout := range want {

+ if got.Rows[host].Layout != layout {

+ t.Errorf("%s = %+v, want %s", host, got.Rows[host], layout)

+ }

+}

+

+// A host mid-move (journal) or held after a refused move (hold marker) is

+// fenced on an unmanaged box too, whatever its row's paths say: the

+// files may be between two directories, and the console's downgrade guard

+// must not read the flat-pathed row as flat.

+func TestUnmanagedLayoutReportFencesAJournalledOrHeldHost(t *testing.T) {

+ resetLayoutGlobals(t)

+ dd := t.TempDir()

+ t.Setenv("CARLOS_DATA_DIR", dd)

+ if err := os.MkdirAll(filepath.Join(dd, "layout"), 0o700); err != nil {

+ t.Fatal(err)

+ }

+ if err := os.WriteFile(provision.LayoutJournalPath(dd, "moving.example.com"), []byte("{}"), 0o600); err != nil {

+ t.Fatal(err)

+ }

+ if err := os.WriteFile(filepath.Join(dd, "layout", "held.example.com.hold"), []byte("{}"), 0o600); err != nil {

+ t.Fatal(err)

+ }

+ flat := func(host string) registry.Route {

+ return registry.Route{Host: host, Kind: "instance", Hibernate: true, DBPath: provision.DefaultDBPath(dd, host), Socket: provision.DefaultSocket(host)}

+ }

+ got := layoutStatusReport([]registry.Route{flat("moving.example.com"), flat("held.example.com"), flat("flat.example.com")})

+ want := map[string]string{"moving.example.com": "fenced", "held.example.com": "fenced", "flat.example.com": "flat"}

+ if len(got.Rows) != len(want) {

+ t.Fatalf("rows = %+v, want %v", got.Rows, want)

+ }

+ for host, layout := range want {

+ if got.Rows[host].Layout != layout {

+ t.Errorf("%s = %+v, want %s", host, got.Rows[host], layout)

+ }

+}

+

+// mintLayoutIntent (ReconcileConfig.Layout for this box): an absent intent

+// object is "" with no error, so mints proceed flat; only a real read

+// failure, or a request this binary cannot read, is an error that defers

+// them.

+func TestMintLayoutIntentAbsentIsFlatAndUnreadableIsAnError(t *testing.T) {

+ ctx := context.Background()

+ s := release.NewDirStore(t.TempDir())

+ if mode, err := mintLayoutIntent(ctx, s); err != nil || mode != "" {

+ t.Fatalf("absent object = %q %v, want \"\" and no error", mode, err)

+ }

+ if _, err := s.Put(ctx, release.LayoutIntentKey(), []byte(`{"schema":2,"requests":[]}`), ""); err != nil {

+ t.Fatal(err)

+ }

+ if _, err := mintLayoutIntent(ctx, s); err == nil {

+ t.Fatal("a newer schema must be an error, so the reconciler defers rather than mints flat")

}

diff --git a/internal/host/layoutleg.go b/internal/host/layoutleg.go

index b993ec4c5..599c10875 100644

--- a/internal/host/layoutleg.go

+++ b/internal/host/layoutleg.go

@@ -9,7 +9,9 @@ package host

import (

"context"

+ "fmt"

"log"

+ "os"

"os/user"

"strconv"

"strings"

@@ -19,6 +21,7 @@ import (

"github.com/carlosframework/platform/internal/activator"

"github.com/carlosframework/platform/internal/layout"

+ "github.com/carlosframework/platform/internal/provision"

"github.com/carlosframework/platform/internal/registry"

"github.com/carlosframework/platform/internal/release"

"github.com/carlosframework/platform/internal/status"

@@ -58,17 +61,24 @@ func noteLayoutIntent(mode string) { layoutIntentSeen.Store(&mode) }

// Classify — the same classification a wake would see, so status never

// says something different from what the activator actually does.

//

-// nil means no activator is armed on this box, so there is no pass and

-// nothing to report; the caller (statusLoop) leaves Probes.Layout nil,

-// which CollectBox carries through as an absent key, never a guessed mode.

+// With no activator armed there is no pass and no Classify, but the box

+// is not silent either: it reports Mode "unmanaged" with every

+// instance-kind row judged by path shape alone (provision.NestedPaths),

+// because the one thing a reader needs to know about such a box is

+// whether a downgrade would strand a nested database, and that is a fact

+// about paths, not about wakes. Nothing wakes a hibernate-flagged row on

+// such a box, so no row there is managed, and the console's downgrade

+// guard (layoutguard.go) treats the report like any other: a row named

+// nested blocks, flat rows do not. Silence from a box at or above the

+// floor would otherwise block its fleet's rollback for good.

//

-// routes is the same list the caller already read for its other probes —

+// routes is the same list the caller already read for its other probes;

// this never lists the registry itself, mirroring every other probe in

// this file.

func layoutStatusReport(routes []registry.Route) *status.LayoutReport {

pass := armedLayoutPass.Load()

if pass == nil {

- return nil

+ return unmanagedLayoutReport(routes)

}

mode := lastLayoutIntent()

if mode == "" {

@@ -101,6 +111,42 @@ func layoutStatusReport(routes []registry.Route) *status.LayoutReport {

return &status.LayoutReport{Mode: mode, Rows: rows}

}

+// LayoutModeUnmanaged is the report Mode of a box with no activator armed:

+// no pass runs there and no hibernate-flagged row ever wakes, so its rows

+// are judged by path shape alone (see layoutStatusReport).

+const LayoutModeUnmanaged = "unmanaged"

+

+// unmanagedLayoutReport is layoutStatusReport's answer with no pass armed.

+// Instance-kind rows only, as the managed report: fenced when a journal or

+// hold marker names the host (a move was in flight, or a refused move kept

+// the host down, when the activator went away, and the files are still

+// where recovery expects them), nested by path shape, external for an

+// unflagged socket route, flat otherwise. The fenced check comes first,

+// as in activator.Classify: a mid-move host's row can still carry flat

+// paths while its files are already between two directories.

+func unmanagedLayoutReport(routes []registry.Route) *status.LayoutReport {

+ rows := map[string]status.RowLayout{}

+ dd := provisionTickConfig(os.Getenv).DataDir

+ for _, rt := range routes {

+ if rt.Kind != "instance" {

+ continue

+ }

+ switch {

+ case layout.JournalPresent(dd, rt.Host):

+ rows[rt.Host] = status.RowLayout{Layout: string(activator.LayoutFenced), Reason: "layout migration in flight; see " + provision.LayoutJournalPath(dd, rt.Host) + " (no activator on this box to recover it)"}

+ case layout.HoldMarkerPresent(dd, rt.Host):

+ rows[rt.Host] = status.RowLayout{Layout: string(activator.LayoutFenced), Reason: "held down after a refused move to nested (no activator on this box to retry it)"}

+ case provision.NestedPaths(dd, rt):

+ rows[rt.Host] = status.RowLayout{Layout: string(activator.LayoutNested), Reason: "no activator on this box; judged by path shape"}

+ case rt.Socket != "" && !rt.Hibernate:

+ rows[rt.Host] = status.RowLayout{Layout: "external"}

+ case rt.Hibernate && rt.Socket != "":

+ rows[rt.Host] = status.RowLayout{Layout: string(activator.LayoutFlat)}

+ }

+ return &status.LayoutReport{Mode: LayoutModeUnmanaged, Rows: rows}

+}

+

// reloadRouting asks the live edge to rebuild its routes now, if there is

// one yet.

func reloadRouting() {

@@ -394,3 +440,21 @@ func logLayoutResults(rs []layout.Result) (retry bool) {

}

return retry || len(awake) > 0

}

+

+// mintLayoutIntent is provision.ReconcileConfig.Layout for this box: the

+// mode of the newest intent addressing its label, "" when none does. An

+// unreadable intent is an error, and the reconciler defers its mints

+// rather than minting flat on a box the fleet may want nested. The same

+// read the layout leg makes, so a mint and a pass agree.

+func mintLayoutIntent(ctx context.Context, store release.Store) (string, error) {

+ in, found, err := release.LoadLayoutIntent(ctx, store, BoxLabel())

+ switch {

+ case err != nil:

+ return "", err

+ case !found:

+ return release.LayoutModeNone, nil

+ case !release.ValidLayoutMode(in.Mode):

+ return "", fmt.Errorf("layout intent %s names mode %q, which this binary does not know", in.ID, in.Mode)

+ }

+ return in.Mode, nil

+}

diff --git a/internal/host/systemapply.go b/internal/host/systemapply.go

index da0441136..e77356955 100644

--- a/internal/host/systemapply.go

+++ b/internal/host/systemapply.go

@@ -375,7 +375,16 @@ func (a *systemApplier) install(ctx context.Context, req systemRequest, p *relea

// order is enforced"). The candidate's own staged unit set is the

// evidence of which kind of release it is. Refused here, before the

// journal, so a refusal leaves the box exactly as it was.

- if !slices.Contains(newUnits, layoutMountUnit) {

+ //

+ // The unit set is not the whole story: v0.20.0 to v0.24.2 ship the

+ // mounts and still cannot wake a nested row (their edge unit keeps

+ // RestrictSUIDSGID=, so the agent's openat2 fails; release.LayoutFloor),

+ // and `carlos system update --file` reaches this guard with no console

+ // check ahead of it. So a candidate whose reported version parses and

+ // is below the floor is refused on the same terms. A version that

+ // cannot be parsed (a local build named by a sha) is judged by its

+ // unit set alone, as before; docs/site/cli.md says so.

+ if !slices.Contains(newUnits, layoutMountUnit) || release.BelowFloor(target, release.LayoutFloor) {

if reasons := a.layoutReversalReasons(); len(reasons) > 0 {

return a.record(req, fmt.Errorf("%w: "+systemApplyLayoutRefusal, errSystemRefused, target, strings.Join(reasons, ", ")))

}

diff --git a/internal/host/systemapply_test.go b/internal/host/systemapply_test.go

index 4327daa2d..1e8c88e71 100644

--- a/internal/host/systemapply_test.go

+++ b/internal/host/systemapply_test.go

@@ -401,11 +401,17 @@ func TestApplyStagesTheCandidateFromARootOwnedCopy(t *testing.T) {

// applyHappy sets up a v1 box with a v2 candidate whose fake bootstrap

// plan adds carlos-new-only.path and drops carlos-old-only.timer.

func applyHappy(t *testing.T) (*applyFixture, systemRequest) {

+ return applyHappyVersion(t, "v2")

+}

+

+// applyHappyVersion is applyHappy with the candidate's version named, for

+// the guards that place a version against release.LayoutFloor.

+func applyHappyVersion(t *testing.T, version string) (*applyFixture, systemRequest) {

f := newApplyFixture(t, "stable", "v1")

- body := []byte("#!/bin/sh\necho carlos v2\n")

- req := f.promote("v2", body, false)

+ body := []byte("#!/bin/sh\necho carlos " + version + "\n")

+ req := f.promote(version, body, false)

staged := filepath.Join(f.a.stagingDir(), "carlos")

- f.answer["version:"+staged] = "carlos v2\n"

+ f.answer["version:"+staged] = "carlos " + version + "\n"

f.answer["plan:"+staged] = `{"enabled":["carlos-edge.service","carlos-adopt.timer","carlos-adopt.path","carlos-stage.path","carlos-restart.path","carlos-system-update.path","carlos-system-recover.service","carlos-new-only.path"],"inert":["carlos-restore-verify.timer"]}`

f.answer["units:"+staged] = "carlos-edge.service carlos-adopt.timer carlos-adopt.path carlos-stage.path carlos-restart.path carlos-system-update.path carlos-system-update.service carlos-system-recover.service carlos-new-only.path"

// The unit v1 owns and v2 drops is on the box, so removing it is a

@@ -1458,6 +1464,54 @@ func TestApplyInstallsAPostLayoutCandidateOverNestedRows(t *testing.T) {

}

+// A candidate that ships the mount units but reports a version below

+// release.LayoutFloor (v0.20.0 to v0.24.2: mounts, but an edge unit on

+// which nested rows cannot wake) is refused over a nested row all the same.

+// This is the box's own line of defence against `carlos system update

+// --file`, which the console's guard does not judge. Flat rows let it in.

+func TestApplyRefusesAMountedButBelowFloorCandidateOverANestedRow(t *testing.T) {

+ for _, tc := range []struct {

+ name string

+ rows []registry.Route

+ want string

+ reason string

+ }{

+ {"nested row", nil, systemOutcomeRefused, "v0.24.2 predates the instance layout, and this box still has: acme.example.com"},

+ {"flat rows", nil, systemOutcomeApplied, ""},

+ } {

+ t.Run(tc.name, func(t *testing.T) {

+ f, req := applyHappyVersion(t, "v0.24.2")

+ f.postLayout()

+ dd := f.a.dataDir

+ if tc.name == "nested row" {

+ f.seedRoutes(flatRow(dd, "flat.example.com"), nestedRow(dd, "acme.example.com"))

+ } else {

+ f.seedRoutes(flatRow(dd, "flat.example.com"), flatRow(dd, "acme.example.com"))

+ }

+ if err := f.a.Run(context.Background(), false); err != nil {

+ t.Fatal(err)

+ }

+ r := f.result()

+ if r == nil || r.Outcome != tc.want || !r.matches("stable", "v0.24.2", req.PromotedAt) {

+ t.Fatalf("result = %+v, want %s", r, tc.want)

+ }

+ if tc.reason != "" && !strings.Contains(r.Reason, tc.reason) {

+ t.Fatalf("reason = %q, want it to contain %q", r.Reason, tc.reason)

+ }

+ })

+ }

+ // At the floor the unit set is what counts, and it ships the mounts.

+ f, req := applyHappyVersion(t, "v0.25.0")

+ f.postLayout()

+ f.seedRoutes(nestedRow(f.a.dataDir, "acme.example.com"))

+ if err := f.a.Run(context.Background(), false); err != nil {

+ t.Fatal(err)

+ }

+ if r := f.result(); r == nil || r.Outcome != systemOutcomeApplied || !r.matches("stable", "v0.25.0", req.PromotedAt) {

+ t.Fatalf("v0.25.0 over a nested row = %+v, want applied", r)

+ }

+}

+

// A registry the guard cannot read cannot vouch that nothing is nested.

func TestApplyRefusesAPreLayoutCandidateWhenTheRegistryWillNotOpen(t *testing.T) {

f, _ := applyHappy(t)

diff --git a/internal/host/units/carlos-instance-control.service b/internal/host/units/carlos-instance-control.service

index 364de03cb..b3eaab0ac 100644

--- a/internal/host/units/carlos-instance-control.service

+++ b/internal/host/units/carlos-instance-control.service

@@ -12,6 +12,16 @@

# main process (SO_PEERCRED uid carlos, pid == MainPID of

# carlos-edge.service): a flat tenant is a child of that pid, never the pid

# itself, and a nested tenant cannot reach the socket at all.

+#

+# At start it also looks UNDER the two instance bind mounts (unshare -m in

+# a private namespace, umount, list; design §8.2, "Recovery for a layout

+# that was unmounted anyway"). Files left in the raw mountpoint directory

+# while the layout was unmounted would otherwise be hidden by the mount and

+# invisible to the layout pass's destination check; a held file is a

+# failing line in every probe report and keeps /ready refusing nested wakes

+# until an operator moves or removes it and restarts this service. Root is

+# what makes that inspection possible: the private namespace and the umount

+# need CAP_SYS_ADMIN. ProtectSystem=strict leaves /data and /run visible.

[Unit]

Description=carlos instance control (root: start/stop carlos-instance*@ units for the agent)

Requires=carlos-instance-control.socket data-instances.mount run-carlos-instances.mount run-carlos-logs.mount

diff --git a/internal/instancectl/server.go b/internal/instancectl/server.go

index 379f6d02d..edc87d99a 100644

--- a/internal/instancectl/server.go

+++ b/internal/instancectl/server.go

@@ -66,12 +66,23 @@ type Server struct {

Statfs func(path string) (avail, total uint64, err error)

DiskEvery time.Duration

+ // Run executes one command as root and returns its combined output;

+ // production is exec.CommandContext, tests record. Only

+ // inspectMountpoints uses it, and every argument it passes is a

+ // constant: nothing from a request reaches it.

+ Run func(ctx context.Context, args ...string) (string, error)

+

// cgroupWaitBudget bounds a wait for an empty cgroup; tests shorten it.

cgroupWaitBudget time.Duration

mu sync.Mutex

report ProbeReport

probed bool

+ // startLines is inspectMountpoints' verdict, taken once at start and

+ // folded into every report RunProbe stores. A failing line keeps the

+ // box not ready whatever the probe says, until an operator clears the

+ // files and restarts the service.

+ startLines []VerdictLine

}

const (

@@ -93,6 +104,10 @@ func Run(args []string) {

out, err := exec.CommandContext(ctx, "systemctl", args...).CombinedOutput()

return string(out), err

},

+ Run: func(ctx context.Context, args ...string) (string, error) {

+ out, err := exec.CommandContext(ctx, args[0], args[1:]...).CombinedOutput()

+ return string(out), err

+ },

PeerOK: productionPeerOK(),

CgroupRoot: "/sys/fs/cgroup",

ProbeOut: filepath.Join(controlsock.Dir, "probe.out"),

@@ -120,6 +135,7 @@ func Run(args []string) {

}

ctx, cancel := context.WithCancel(context.Background())

defer cancel()

+ s.InspectMountpoints(ctx)

go s.probeLoop(ctx)

go s.diskLoop(ctx)

if err := s.Serve(ctx, ln); err != nil {

@@ -663,5 +679,85 @@ func (s *Server) RunProbe(ctx context.Context) {

s.logf("instance-control: probe: %s %s %s", l.Check, l.Verdict, l.Detail)

}

- set(r)

+ set(s.withStartLines(r))

+}

+

+// mountpoints are the two bind mounts whose underlying directories

+// inspectMountpoints looks under (design §8.2, "Recovery for a layout that

+// was unmounted anyway").

+var mountpoints = []struct{ name, where string }{

+ {"data-instances", "/data/instances"},

+ {"run-carlos-instances", "/run/carlos/instances"},

+}

+

+// InspectMountpoints runs inspectMountpoints once and keeps its lines for

+// every later report. Run calls it before the probe loop starts.

+func (s *Server) InspectMountpoints(ctx context.Context) {

+ lines := s.inspectMountpoints(ctx)

+ for _, l := range lines {

+ if l.Verdict != "ok" {

+ s.logf("instance-control: %s %s %s", l.Check, l.Verdict, l.Detail)

+ }

+ s.mu.Lock()

+ s.startLines = lines

+ s.mu.Unlock()

+}

+

+// inspectMountpoints looks UNDER the two bind mounts: in a private mount

+// namespace it unmounts each bind and lists the raw mountpoint directory.

+// Anything there was written while the layout was unmounted (an operator

+// removing units by hand, a boot without the mounts), and the layout

+// pass's destination check would otherwise never see it: the mount hides

+// it, and the old binary would restore a second database over it on the

+// next unmount. The verdict is taken once at start, as root, because

+// unmounting in a private namespace needs CAP_SYS_ADMIN; it clears only

+// when an operator moves or removes the files and restarts the service.

+// An inspection that cannot run (no unshare, no permission) is

+// unverified, and does not block readiness on its own: the probe's own

+// checks stand, and the unverified line is in every report for the

+// operator to see.

+func (s *Server) inspectMountpoints(ctx context.Context) []VerdictLine {

+ if s.Run == nil {

+ return nil

+ }

+ var lines []VerdictLine

+ for _, m := range mountpoints {

+ // The listing runs after the umount so it reads the underlying

+ // directory; a umount that fails because nothing is mounted there

+ // is fine (the listing is then of the directory itself, which is

+ // what an unmounted layout looks like), so its error is dropped.

+ out, err := s.Run(ctx, "unshare", "-m", "--propagation", "private", "sh", "-c",

+ "umount "+m.where+" 2>/dev/null; ls -A "+m.where)

+ if err != nil {

+ lines = append(lines, VerdictLine{Check: "mountpoint-" + m.name, Verdict: "unverified", Detail: strings.TrimSpace(out + " " + err.Error())})

+ continue

+ }

+ if n := len(strings.Fields(out)); n > 0 {

+ lines = append(lines, VerdictLine{Check: "mountpoint-" + m.name, Verdict: "fail",

+ Detail: fmt.Sprintf("mountpoint held files while unmounted: %d entries under %s; move or remove them, then restart carlos-instance-control", n, m.where)})

+ continue

+ }

+ lines = append(lines, VerdictLine{Check: "mountpoint-" + m.name, Verdict: "ok"})

+ }

+ return lines

+}

+

+// withStartLines prepends the start inspection to a probe report and

+// recomputes Ready: the probe's own verdict, and no start line failing.

+// A copy, so the stored report never aliases the probe's slice.

+func (s *Server) withStartLines(r ProbeReport) ProbeReport {

+ s.mu.Lock()

+ start := append([]VerdictLine(nil), s.startLines...)

+ s.mu.Unlock()

+ if len(start) == 0 {

+ return r

+ }

+ out := ProbeReport{Ready: r.Ready, Lines: append(start, r.Lines...)}

+ for _, l := range start {

+ if l.Verdict == "fail" {

+ out.Ready = false

+ }

+ return out

}

diff --git a/internal/instancectl/server_test.go b/internal/instancectl/server_test.go

index 13bc53db1..aaf822f57 100644

--- a/internal/instancectl/server_test.go

+++ b/internal/instancectl/server_test.go

@@ -529,3 +529,104 @@ func TestWaitCgroupEmptyTreatsAnAbsentCgroupAsEmpty(t *testing.T) {

t.Errorf("absent cgroup: %v", err)

}

+

+// fakeRun answers inspectMountpoints' unshare invocations by the

+// mountpoint named in the command: the listing under each raw directory,

+// or an exec failure.

+type fakeRun struct {

+ listings map[string]string // "/data/instances" -> ls -A output

+ err error

+ calls []string

+}

+

+func (f *fakeRun) run(_ context.Context, args ...string) (string, error) {

+ line := strings.Join(args, " ")

+ f.calls = append(f.calls, line)

+ if f.err != nil {

+ return "", f.err

+ }

+ for where, out := range f.listings {

+ if strings.Contains(line, "ls -A "+where) {

+ return out, nil

+ }

+ return "", nil

+}

+

+func probeOK(t *testing.T, s *Server) {

+ t.Helper()

+ b, _ := json.Marshal(ProbeReport{Ready: true, Lines: []VerdictLine{{Check: "pid1", Verdict: "ok"}}})

+ if err := os.WriteFile(s.ProbeOut, b, 0o644); err != nil {

+ t.Fatal(err)

+ }

+ s.RunProbe(context.Background())

+}

+

+// Both raw directories empty: two ok lines, and readiness follows the

+// probe alone.

+func TestMountpointInspectionCleanIsReady(t *testing.T) {

+ run := &fakeRun{}

+ s, c := newServer(t, &fakeCtl{}, func(s *Server) { s.Run = run.run })

+ s.InspectMountpoints(context.Background())

+ probeOK(t, s)

+ if resp, err := c.Get("http://unix/ready"); err != nil || resp.StatusCode != 204 {

+ t.Fatalf("ready = %v %v, want 204", resp, err)

+ }

+ rep, ready := s.Ready()

+ if !ready || len(rep.Lines) != 3 || rep.Lines[0].Check != "mountpoint-data-instances" || rep.Lines[0].Verdict != "ok" || rep.Lines[1].Check != "mountpoint-run-carlos-instances" {

+ t.Fatalf("report = %+v", rep)

+ }

+ if len(run.calls) != 2 || !strings.HasPrefix(run.calls[0], "unshare -m --propagation private sh -c umount /data/instances") {

+ t.Errorf("calls = %q", run.calls)

+ }

+}

+

+// Files under the raw /data/instances: the report carries a failing

+// mountpoint line and /ready is 503 even after a passing probe, on every

+// probe after it.

+func TestMountpointHeldFilesBlockReadiness(t *testing.T) {

+ run := &fakeRun{listings: map[string]string{"/data/instances": "a.example.com\nb.example.com\n"}}

+ s, c := newServer(t, &fakeCtl{}, func(s *Server) { s.Run = run.run })

+ s.InspectMountpoints(context.Background())

+ for i := 0; i < 2; i++ {

+ probeOK(t, s)

+ resp, err := c.Get("http://unix/ready")

+ if err != nil || resp.StatusCode != 503 {

+ t.Fatalf("probe %d: ready = %v %v, want 503", i, resp, err)

+ }

+ var got ProbeReport

+ if err := json.NewDecoder(resp.Body).Decode(&got); err != nil || got.Ready {

+ t.Fatalf("body = %+v %v", got, err)

+ }

+ l := got.Lines[0]

+ if l.Check != "mountpoint-data-instances" || l.Verdict != "fail" || !strings.Contains(l.Detail, "mountpoint held files while unmounted: 2 entries under /data/instances") {

+ t.Errorf("line = %+v", l)

+ }

+ if got.Lines[1].Verdict != "ok" || got.Lines[2].Check != "pid1" {

+ t.Errorf("lines = %+v", got.Lines)

+ }

+}

+

+// unshare cannot run: both lines are unverified, in the report, and

+// readiness follows the probe alone.

+func TestMountpointInspectionUnverifiedDoesNotBlock(t *testing.T) {

+ run := &fakeRun{err: errors.New("exec: \"unshare\": executable file not found in $PATH")}

+ s, c := newServer(t, &fakeCtl{}, func(s *Server) { s.Run = run.run })

+ s.InspectMountpoints(context.Background())

+ probeOK(t, s)

+ if resp, err := c.Get("http://unix/ready"); err != nil || resp.StatusCode != 204 {

+ t.Fatalf("ready = %v %v, want 204", resp, err)

+ }

+ rep, _ := s.Ready()

+ if len(rep.Lines) != 3 || rep.Lines[0].Verdict != "unverified" || !strings.Contains(rep.Lines[0].Detail, "unshare") {

+ t.Fatalf("report = %+v", rep)

+ }

+ // A failing probe is still a failing probe.

+ b, _ := json.Marshal(ProbeReport{Ready: false, Lines: []VerdictLine{{Check: "pid1", Verdict: "fail"}}})

+ os.WriteFile(s.ProbeOut, b, 0o644)

+ s.RunProbe(context.Background())

+ if resp, _ := c.Get("http://unix/ready"); resp.StatusCode != 503 {

+ t.Error("a failing probe read as ready")

+ }

+}

diff --git a/internal/layout/pass.go b/internal/layout/pass.go

index b4e23361a..be711ffca 100644

--- a/internal/layout/pass.go

+++ b/internal/layout/pass.go

@@ -436,6 +436,15 @@ func removeHoldMarker(dataDir, host string) {

}

+// HoldMarkerPresent reports whether host has a hold marker, without reading

+// it. Lstat, like JournalPresent, and an error other than "not exist"

+// counts as present: a marker this process cannot stat is not proof there

+// is none.

+func HoldMarkerPresent(dataDir, host string) bool {

+ _, err := os.Lstat(filepath.Join(journalDir(dataDir), holdMarkerName(host)))

+ return !errors.Is(err, os.ErrNotExist)

+}

+

// HoldMarkers lists the hosts with a hold marker in the layout directory.

func HoldMarkers(dataDir string) ([]string, error) {

entries, err := os.ReadDir(journalDir(dataDir))

diff --git a/internal/provision/reconcile.go b/internal/provision/reconcile.go

index 123e09d96..75d64b1f5 100644

--- a/internal/provision/reconcile.go

+++ b/internal/provision/reconcile.go

@@ -59,6 +59,17 @@ type ReconcileConfig struct {

// both gates below compare this field with exact string equality

// against InboundNone, so an un-folded value would defeat them silently.

Inbound string

+ // Layout answers which layout this box's intent names, "" or

+ // release.LayoutModeFlat for flat and release.LayoutModeNested for

+ // nested, read from the deploy store the pass already has in hand

+ // (isolated-exec design §8.6 step 5, brought forward in plan 5): a row

+ // minted on a nested box is minted at the nested paths, never flat

+ // and then migrated, so a new tenant never runs flat beside nested

+ // siblings, not even for one pass. An error defers every mint on this

+ // pass rather than guessing: a box that cannot tell its layout must

+ // not mint a flat row the fleet may want nested. Nil is flat (tests,

+ // and a caller with no intent to read).

+ Layout func(ctx context.Context, s release.Store) (string, error)

// DryRun reports what the pass would do and writes nothing — not to the

// registry, not anywhere. The backfill mitigation (design §4.3): a

// production route is adopted only after someone has read what adopting

@@ -374,7 +385,13 @@ func (cfg ReconcileConfig) inScope(region, pool string, everywhere bool) bool {

// exclusion set from `r.Hibernate && r.DBPath != ""`, so a hibernating route

// whose row carried no path would be swept up by the dataDir glob and

// replicated box-wide as if it never slept.

-func routeFor(rec Record, dataDir, seed string) registry.Route {

+//

+// nested mints the derived paths at the nested layout's helpers instead

+// (InstanceDir, NestedSocket): the shape the layout pass would otherwise

+// move a flat row to. Only the mint branch passes true, from

+// ReconcileConfig.Layout; the drift and adoption comparisons derive flat

+// and judge a nested row by NestedPaths.

+func routeFor(rec Record, dataDir, seed string, nested bool) registry.Route {

rt := registry.Route{

Host: rec.Host, Slug: rec.Slug, Kind: rec.KindOrDefault(), Unit: rec.Unit,

App: rec.App, Account: rec.Account, Pool: rec.Pool, Region: rec.Region,

@@ -389,11 +406,42 @@ func routeFor(rec Record, dataDir, seed string) registry.Route {

if rec.DerivesPaths() {

rt.Socket = DefaultSocket(rec.Host)

rt.DBPath = DefaultDBPath(dataDir, rec.Host)

+ // Only an activator-managed row has a nested shape: an unflagged

+ // socket route is another supervisor's process, whose paths stay

+ // flat in every release (design §8.2 step 1).

+ if nested && rt.Hibernate {

+ rt.Socket = NestedSocket(rec.Host)

+ rt.DBPath = NestedDBPath(dataDir, rec.Host)

+ }

}

rt.Addr, rt.Addrs, rt.Health = upstreamCols(rec)

return rt

}

+// mintLayoutReader answers "mint nested?" for one pass, asking cfg.Layout

+// at most once and only when a mint needs the answer. A nil Layout is

+// flat. The answer is cached error and all: a store that failed once this

+// pass is not asked again per record.

+func mintLayoutReader(ctx context.Context, s release.Store, cfg ReconcileConfig) func() (bool, error) {

+ var (

+ asked bool

+ nested bool

+ err error

+ )

+ return func() (bool, error) {

+ if cfg.Layout == nil {

+ return false, nil

+ }

+ if !asked {

+ asked = true

+ var mode string

+ mode, err = cfg.Layout(ctx, s)

+ nested = err == nil && mode == release.LayoutModeNested

+ }

+ return nested, err

+ }

+}

+

// upstreamCols renders a record's THREE upstream columns exactly as a minted

// route carries them: the legacy single addr, the comma-joined addr set, and

// the probe-config JSON.

@@ -988,6 +1036,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg

for _, rt := range routes {

byHost[rt.Host] = rt

}

+ mintLayout := mintLayoutReader(ctx, s, cfg)

for _, rec := range recs {

if trashedFor(rec.Account, rec.App) {

@@ -1022,7 +1071,16 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg

// exact route this branch hardcoded before they existed — that is

// the additivity rule, pinned by TestRecordWireShapeIsAdditive and

// TestReconcileMints.

- nr := routeFor(rec, cfg.DataDir, seed)

+ // The box's layout decides the paths a fresh row is minted at.

+ // Read once per pass, lazily, so a pass with nothing to mint

+ // never reads it; a read that fails defers every mint on this

+ // pass (ReconcileConfig.Layout).

+ nested, lerr := mintLayout()

+ if lerr != nil {

+ log.Printf("provision: %s: not minting this pass: the box's layout intent could not be read: %v", rec.Host, lerr)

+ continue

+ }

+ nr := routeFor(rec, cfg.DataDir, seed, nested)

// A box that cannot provide this instance's config must not

// admit it: it would start without its secrets, and a keyed

// sibling would be left with a route that lost the race to one

@@ -1124,7 +1182,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg

// by design (the layout pass moved them), so when that is the

// only difference the line says so, rather than pointing an

// operator at `carlos remove`.

- want := routeFor(rec, cfg.DataDir, seed)

+ want := routeFor(rec, cfg.DataDir, seed, false)

unitDrift := rec.Unit != "" && want.Unit != existing.Unit

if want.Kind != existing.Kind || unitDrift ||

want.Socket != existing.Socket || want.DBPath != existing.DBPath {

@@ -1148,7 +1206,10 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg

// as before: adoption grants this pass's removal phase authority

// over the row, so the match test is the only thing standing

// between a backfill and a retired production route.

- want := routeFor(rec, cfg.DataDir, seed)

+ // A hand-added row on a nested box (`carlos add` there mints

+ // nested too) matches the record at the nested paths; the row's

+ // own shape says which derivation to compare against.

+ want := routeFor(rec, cfg.DataDir, seed, NestedPaths(cfg.DataDir, existing))

if reason := shapeMismatch(existing, want); reason != "" {

log.Printf("provision: %s exists but is not the route this record describes (%s) — leaving it alone",

rec.Host, reason)

diff --git a/internal/provision/reconcile_test.go b/internal/provision/reconcile_test.go

index 4067c63ec..624517255 100644

--- a/internal/provision/reconcile_test.go

+++ b/internal/provision/reconcile_test.go

@@ -1072,7 +1072,7 @@ func TestReconcileConvergesLegacyAddrToAddrs(t *testing.T) {

// (shapeMismatch is the adoption gate AND the drift test — the two failures

// the un-cleared addr caused, asserted at their source).

seed := seedJSON(&Manifest{Seed: []string{"seed"}}) // setup()'s manifest

- if reason := shapeMismatch(after, routeFor(rec, cfg.DataDir, seed)); reason != "" {

+ if reason := shapeMismatch(after, routeFor(rec, cfg.DataDir, seed, false)); reason != "" {

t.Errorf("converged row still diverges from its record (%s) — the pass would log this every tick and refuse re-adoption", reason)

}

@@ -2655,7 +2655,7 @@ func TestEverywhereClassAgreesAcrossRecordAndRow(t *testing.T) {

t.Fatalf("fixture is not a shape this pass would ever see: %v", err)

}

fromRecord := everywhereClass(f.rec)

- fromRow := everywhereRoute(routeFor(f.rec, "/data", `["seed"]`))

+ fromRow := everywhereRoute(routeFor(f.rec, "/data", `["seed"]`, false))

if fromRecord != fromRow {

t.Fatalf("class disagrees across the mint and removal gates: everywhereClass = %v, "+

"everywhereRoute(routeFor(...)) = %v — a record this box mints must be a row it can retire",

@@ -2674,7 +2674,7 @@ func TestEverywhereClassAgreesAcrossRecordAndRow(t *testing.T) {

func TestRouteForCarriesAddrsAndHealth(t *testing.T) {

rec := Record{Host: "m.example.com", App: "web", Account: "bab", Channel: "stable",

Kind: "service", Addrs: []string{"10.0.0.1:3000", "10.0.0.2:3000"}}

- rt := routeFor(rec, "/data", "seed")

+ rt := routeFor(rec, "/data", "seed", false)

if rt.Addrs != "10.0.0.1:3000,10.0.0.2:3000" {

t.Fatalf("Addrs = %q", rt.Addrs)

}

@@ -2687,7 +2687,7 @@ func TestRouteForCarriesAddrsAndHealth(t *testing.T) {

}

// Legacy single-addr record: byte-identical route to before this change.

old := Record{Host: "s.example.com", App: "web", Account: "bab", Channel: "stable", Kind: "service", Addr: "10.0.0.9:3000"}

- ort := routeFor(old, "/data", "seed")

+ ort := routeFor(old, "/data", "seed", false)

if ort.Addrs != "" || ort.Health != "" || ort.Addr != "10.0.0.9:3000" {

t.Fatalf("legacy record's route changed shape: %+v", ort)

}

@@ -2703,7 +2703,7 @@ func TestRouteForMaterializesPartialHealthDefaults(t *testing.T) {

rec := Record{Host: "m.example.com", App: "web", Account: "bab", Channel: "stable",

Kind: "service", Addrs: []string{"10.0.0.1:3000", "10.0.0.2:3000"},

Health: &Health{Path: "/healthz"}}

- rt := routeFor(rec, "/data", "seed")

+ rt := routeFor(rec, "/data", "seed", false)

var h Health

if err := json.Unmarshal([]byte(rt.Health), &h); err != nil {

t.Fatalf("health JSON = %q (%v)", rt.Health, err)

@@ -2851,11 +2851,11 @@ func TestReconcileUnsetInboundMintsAsBefore(t *testing.T) {

// onto the route, byte for byte — no derivation, unlike Hibernate/StatusNamed.

func TestRouteForCarriesSteering(t *testing.T) {

rec := Record{Host: "m.example.com", App: "web", Account: "bab", Channel: "stable", Steering: "latency"}

- if got := routeFor(rec, "/data", "seed").Steering; got != "latency" {

+ if got := routeFor(rec, "/data", "seed", false).Steering; got != "latency" {

t.Fatalf("Steering = %q, want latency", got)

}

unsteered := Record{Host: "s.example.com", App: "web", Account: "bab", Channel: "stable"}

- if got := routeFor(unsteered, "/data", "seed").Steering; got != "" {

+ if got := routeFor(unsteered, "/data", "seed", false).Steering; got != "" {

t.Fatalf("Steering = %q, want empty for a record that never mentioned it", got)

}

@@ -3050,3 +3050,103 @@ func TestReplicaNeverEverywhereClass(t *testing.T) {

t.Error("everywhereRoute(replica row) = true — the removal-phase half of the same question")

}

+

+// A fresh row on a box whose layout intent is nested is minted at the

+// nested paths (design §8.6 step 5, plan 5), never flat and then moved. A

+// flat or absent intent mints the flat defaults as before; an unflagged

+// (externally supervised) socket route stays flat whatever the intent; an

+// intent that cannot be read defers the mint rather than guessing.

+func TestReconcileMintsNestedUnderANestedIntent(t *testing.T) {

+ s, sqid := setup(t)

+ off := false

+ recs := []Record{

+ acmeRecord(sqid),

+ {Schema: 1, Host: "router.go.tito.io", Slug: "router", App: "tito", Account: sqid,

+ Pool: "tito", Region: "ie", Channel: "stable", Hibernate: &off, CreatedAt: time.Now()},

+ }

+ for _, rec := range recs {

+ if err := CreateRecord(ctx, s, rec); err != nil {

+ t.Fatal(err)

+ }

+ reg := regWith(t)

+ cfg := ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data",

+ Layout: func(context.Context, release.Store) (string, error) { return release.LayoutModeNested, nil }}

+ if err := Reconcile(ctx, s, reg, cfg); err != nil {

+ t.Fatal(err)

+ }

+ got := *mustGet(t, reg, "acme.go.tito.io")

+ if got.DBPath != NestedDBPath("/data", "acme.go.tito.io") || got.Socket != NestedSocket("acme.go.tito.io") || !got.Hibernate {

+ t.Errorf("minted under a nested intent = db %q socket %q hibernate %v, want the nested paths", got.DBPath, got.Socket, got.Hibernate)

+ }

+ ext := *mustGet(t, reg, "router.go.tito.io")

+ if ext.DBPath != DefaultDBPath("/data", "router.go.tito.io") || ext.Socket != DefaultSocket("router.go.tito.io") {

+ t.Errorf("an unflagged socket route minted nested: db %q socket %q", ext.DBPath, ext.Socket)

+ }

+ // The next pass sees the nested row as the record's own, not drift,

+ // and converges nothing on it.

+ if err := Reconcile(ctx, s, reg, cfg); err != nil {

+ t.Fatal(err)

+ }

+ if again := *mustGet(t, reg, "acme.go.tito.io"); again.DBPath != got.DBPath || again.Socket != got.Socket {

+ t.Errorf("second pass moved the row: %+v", again)

+ }

+

+ // Flat intent, and no intent: the flat defaults.

+ for name, layout := range map[string]func(context.Context, release.Store) (string, error){

+ "flat": func(context.Context, release.Store) (string, error) { return release.LayoutModeFlat, nil },

+ "none": func(context.Context, release.Store) (string, error) { return release.LayoutModeNone, nil },

+ "nil": nil,

+ } {

+ reg := regWith(t)

+ if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data", Layout: layout}); err != nil {

+ t.Fatal(err)

+ }

+ if got := *mustGet(t, reg, "acme.go.tito.io"); got.DBPath != "/data/acme.go.tito.io.db" {

+ t.Errorf("%s intent: db %q, want the flat default", name, got.DBPath)

+ }

+

+ // Unreadable: nothing is minted this pass, and the pass itself is not

+ // an error (the records are retried next pass).

+ reg = regWith(t)

+ calls := 0

+ cfg.Layout = func(context.Context, release.Store) (string, error) { calls++; return "", errors.New("store down") }

+ if err := Reconcile(ctx, s, reg, cfg); err != nil {

+ t.Fatal(err)

+ }

+ if rt, err := reg.Get("acme.go.tito.io"); err != nil || rt != nil {

+ t.Errorf("a row was minted while the intent was unreadable: %+v %v", rt, err)

+ }

+ if calls != 1 {

+ t.Errorf("the intent was read %d times in one pass, want once", calls)

+ }

+}

+

+// A hand-added row at the nested paths (`carlos add` on a nested box) is

+// the record's route and is adopted, exactly as a flat hand-added one is.

+func TestReconcileAdoptsAHandAddedNestedRoute(t *testing.T) {

+ s, sqid := setup(t)

+ rec := acmeRecord(sqid)

+ rec.Channel = "edge"

+ if err := CreateRecord(ctx, s, rec); err != nil {

+ t.Fatal(err)

+ }

+ handAdded := registry.Route{

+ Host: "acme.go.tito.io", Slug: "acme", Kind: "instance",

+ Socket: NestedSocket("acme.go.tito.io"), DBPath: NestedDBPath("/data", "acme.go.tito.io"),

+ App: "tito", Account: sqid, Pool: "tito", Region: "ie",

+ Channel: "stable", Hibernate: true, Status: "ready",

+ }

+ reg := regWith(t, handAdded)

+ if err := Reconcile(ctx, s, reg, ReconcileConfig{Region: "ie", Pools: []string{"tito"}, DataDir: "/data"}); err != nil {

+ t.Fatal(err)

+ }

+ got := *mustGet(t, reg, "acme.go.tito.io")

+ if !got.Provisioned || got.Channel != "edge" {

+ t.Fatalf("a nested hand-added route was not adopted: %+v", got)

+ }

+ if got.DBPath != handAdded.DBPath || got.Socket != handAdded.Socket {

+ t.Errorf("adoption rewrote the nested paths: %+v", got)

+ }

+}

diff --git a/internal/release/layoutintent.go b/internal/release/layoutintent.go

index 95c4bc632..f488f603a 100644

--- a/internal/release/layoutintent.go

+++ b/internal/release/layoutintent.go

@@ -20,6 +20,7 @@ import (

"fmt"

"io"

"os"

+ "strings"

"time"

)

@@ -109,6 +110,26 @@ func LoadLayoutIntent(ctx context.Context, s Store, boxLabel string) (*LayoutInt

return nil, false, nil

}

+// LoadLayoutIntents reads the whole log, every scope. found=false means

+// the object is absent. The console door's status read uses it to show the

+// newest request whatever box it addresses; a box reads LoadLayoutIntent,

+// which answers for its own label.

+func LoadLayoutIntents(ctx context.Context, s Store) (*LayoutIntents, bool, error) {

+ key := LayoutIntentKey()

+ raw, _, err := s.Get(ctx, key)

+ if errors.Is(err, os.ErrNotExist) {

+ return nil, false, nil

+ }

+ if err != nil {

+ return nil, false, err

+ }

+ g, err := parseLayoutIntents(key, raw)

+ if err != nil {

+ return nil, false, err

+ }

+ return g, true, nil

+}

+

// AppendLayoutIntent records one request, oldest-first, after every request

// already in the log, and returns the stamped request it wrote. ID and

// RequestedAt are stamped here when the caller leaves them empty, so a

@@ -170,3 +191,56 @@ func AppendLayoutIntent(ctx context.Context, s Store, in LayoutIntent) (LayoutIn

}

return LayoutIntent{}, fmt.Errorf("layout intent: %s is being written by someone else; nothing was requested; try again: %w", key, lastErr)

}

+

+// LayoutFloor is the first platform release on which a nested row WAKES:

+// the one cut from a main that contains plan 4 (7b7d3026), whose edge unit

+// drops RestrictSUIDSGID= so the agent's openat2 calls work. v0.20.0 to

+// v0.24.2 carry the instance mount units, so a nested row's data is safe on

+// them, but the agent cannot open a nested directory there and the row

+// never wakes, which is an outage; so the floor is the release that works,

+// not the release that merely keeps the files. No tag contains 7b7d3026 at

+// the time of writing (latest v0.24.2), so this names the next: v0.25.0

+// MUST be cut from a main that contains 7b7d3026.

+//

+// Both console system doors refuse a pointer move below this floor while

+// any box could hold a nested row, and the layout door refuses `migrate`

+// while any box in scope runs below it. The box-side guard

+// (host/systemapply.go) judges the candidate's staged unit set, and, when

+// the box holds nested rows, the candidate's own reported version against

+// this floor. TestLayoutFloorIsTheFirstReleaseWhereNestedRowsWake pins it.

+var LayoutFloor = SemVer{Major: 0, Minor: 25, Patch: 0}

+

+// ParseReleaseVersion parses a release tag with or without its leading v.

+func ParseReleaseVersion(version string) (SemVer, bool) {

+ return ParseSemVer(strings.TrimPrefix(version, "v"))

+}

+

+// BelowFloor reports whether version parses and is below floor. A tag that

+// does not parse is false: the caller decides what an unplaceable version

+// means for its own rule.

+func BelowFloor(version string, floor SemVer) bool {

+ v, ok := ParseReleaseVersion(version)

+ return ok && v.Compare(floor) < 0

+}

+

+// PreLayout reports whether a release tag predates the layout. A tag that

+// does not parse as a version is treated as pre-layout: refusing is the

+// safe answer for a release nobody can place against the floor, and the

+// refusal names the tag so an operator can see why.

+func PreLayout(version string) bool {

+ v, ok := ParseReleaseVersion(version)

+ if !ok {

+ return true

+ }

+ return v.Compare(LayoutFloor) < 0

+}

+

+// RecordLayoutIntent is the console door's writer (plan 5): one request,

+// stamped here, appended under CAS by AppendLayoutIntent. force asks the

+// boxes in scope to stop an awake host for its move at once rather than

+// wait out its awake grace (LayoutIntent.Force). actor is recorded by the

+// console's own audit row, not in the object: the object is read by every

+// box, and an operator's address has no business in a box's view.

+func RecordLayoutIntent(ctx context.Context, s Store, mode, box string, force bool) (LayoutIntent, error) {

+ return AppendLayoutIntent(ctx, s, LayoutIntent{Mode: mode, Scope: LayoutScope{Box: box}, Force: force})

+}

diff --git a/internal/release/layoutintent_test.go b/internal/release/layoutintent_test.go

index f5c9b5834..fdd32e88b 100644

--- a/internal/release/layoutintent_test.go

+++ b/internal/release/layoutintent_test.go

@@ -96,3 +96,56 @@ func TestAppendLayoutIntentRefusesAnInvalidMode(t *testing.T) {

t.Fatalf("a refused append must leave the object absent: found=%v err=%v", found, err)

}

+

+// TestLayoutFloorIsTheFirstReleaseWhereNestedRowsWake pins the console's

+// downgrade floor to the release history. v0.20.0 is the first tag whose

+// unit set carries data-instances.mount (git ls-tree v0.19.0 and v0.20.0

+// internal/host/units/, checked 2026-10-06), but a nested row cannot wake

+// on v0.20.0 to v0.24.2: their carlos-edge.service still carries

+// RestrictSUIDSGID=, which makes the agent's openat2 fail with ENOSYS, and

+// every nested open goes through it. Plan 4 (7b7d3026) dropped the line;

+// no tag contains it yet, so the floor is the next release, v0.25.0, which

+// must be cut from a main that contains 7b7d3026. Moving either constant

+// means re-checking the tags, not editing this test to match.

+func TestLayoutFloorIsTheFirstReleaseWhereNestedRowsWake(t *testing.T) {

+ if LayoutFloor != (SemVer{Major: 0, Minor: 25, Patch: 0}) {

+ t.Fatalf("LayoutFloor = %+v, want v0.25.0 (the first release carrying 7b7d3026's edge openat2 fix)", LayoutFloor)

+ }

+ for v, pre := range map[string]bool{

+ "v0.19.0": true, "v0.20.0": true, "v0.24.2": true, "0.24.9": true,

+ "v0.25.0": false, "v0.26.0": false, "v1.0.0": false,

+ // Not a version: nobody can place it against the floor, so it is refused.

+ "dev": true, "": true, "v0.25": true, "abc123": true,

+ } {

+ if got := PreLayout(v); got != pre {

+ t.Errorf("PreLayout(%q) = %v, want %v", v, got, pre)

+ }

+ // BelowFloor is the placeable half of PreLayout: an unparseable tag is

+ // false, and the caller decides what that means for its own rule.

+ for v, below := range map[string]bool{"v0.19.0": true, "v0.24.2": true, "v0.25.0": false, "dev": false, "": false} {

+ if got := BelowFloor(v, LayoutFloor); got != below {

+ t.Errorf("BelowFloor(%q, LayoutFloor) = %v, want %v", v, got, below)

+ }

+}

+

+func TestRecordLayoutIntentAppendsWithForceAndScope(t *testing.T) {

+ s := NewDirStore(t.TempDir())

+ ctx := context.Background()

+ first, err := RecordLayoutIntent(ctx, s, LayoutModeNested, "kass-1", true)

+ if err != nil || first.ID == "" || first.Mode != LayoutModeNested || first.Scope.Box != "kass-1" || !first.Force {

+ t.Fatalf("first = %+v %v", first, err)

+ }

+ second, err := RecordLayoutIntent(ctx, s, LayoutModeFlat, "", false)

+ if err != nil || second.ID == first.ID || second.Mode != LayoutModeFlat || second.Scope.Box != "" || second.Force {

+ t.Fatalf("second = %+v %v", second, err)

+ }

+ got, found, err := LoadLayoutIntent(ctx, s, "kass-1")

+ if err != nil || !found || got.ID != second.ID {

+ t.Fatalf("kass-1 sees %+v (found %v, err %v), want the newest request %s", got, found, err, second.ID)

+ }

+ if _, err := RecordLayoutIntent(ctx, s, "sideways", "", false); err == nil {

+ t.Fatal("bad mode accepted")

+ }

+}

diff --git a/status_instances.go b/status_instances.go

index f8ad8ca78..93c9404ec 100644

--- a/status_instances.go

+++ b/status_instances.go

@@ -23,6 +23,7 @@ import (

"net/http"

"net/url"

"sort"

+ "strings"

)

// statusInstanceRow is one awake instance, as the console answers it

@@ -239,15 +240,50 @@ func printStatusLayout(w io.Writer, boxes []statusLayoutBox) {

for _, l := range lines {

fmt.Fprintf(w, "%s %s %s %s\n", l.box, l.host, l.layout, dashIfEmpty(l.reason))

}

- // A box with nested rows beside flat, foreign or external ones is the

- // loud case: each of those still shares the agent's uid and mount

- // namespace, so the nested rows on that box are not isolated from it

- // yet.

+ // A box with flat, foreign or external rows beside nested ones, or

+ // under a nested intent, is the loud case: each of those still shares

+ // the agent's uid and mount namespace, so the nested rows on that box

+ // are not isolated from it yet.

for _, b := range boxes {

if b.Mixed {

fmt.Fprintf(w, "\nWARNING: %s runs nested instances beside flat, foreign or external ones, which share the agent's user and its view of /data. Nested isolation on this box is not in force until none of those remains.\n", b.Box)

}

+ // Boxes that disagree about their mode are the other loud case: a

+ // box-scoped intent covers the boxes it names, and an instance placed

+ // or relocated onto a box outside it is minted flat there (#126, the

+ // Region HA 1b note). Said once, naming both sides.

+ if nested, other := layoutModeSplit(boxes); len(nested) > 0 && len(other) > 0 {

+ fmt.Fprintf(w, "\nWARNING: %s nested while %s not: an instance placed or moved onto a box that is not nested starts flat there. Run carlos system layout migrate without --box once every box is ready.\n",

+ boxList(nested), boxList(other))

+ }

+}

+

+// layoutModeSplit sorts the boxes by mode: nested, and anything else

+// (flat, none, unreadable). Stale boxes are left out: their mode is what

+// they last said, not what they are.

+func layoutModeSplit(boxes []statusLayoutBox) (nested, other []string) {

+ for _, b := range boxes {

+ if b.Stale {

+ continue

+ }

+ if b.Mode == "nested" {

+ nested = append(nested, b.Box)

+ } else {

+ other = append(other, b.Box)

+ }

+ sort.Strings(nested)

+ sort.Strings(other)

+ return nested, other

+}

+

+// boxList is "kass-1 is" or "kass-1 and kass-2 are", for a sentence.

+func boxList(labels []string) string {

+ if len(labels) == 1 {

+ return labels[0] + " is"

+ }

+ return strings.Join(labels[:len(labels)-1], ", ") + " and " + labels[len(labels)-1] + " are"

}

// cpuCell renders a fraction of one core. A nil reading prints as a dash: the

diff --git a/status_instances_test.go b/status_instances_test.go

index 23b2ddd82..b20239db4 100644

--- a/status_instances_test.go

+++ b/status_instances_test.go

@@ -153,3 +153,29 @@ func TestStatusLayoutListDecodesBoxes(t *testing.T) {

t.Fatalf("boxes = %+v", boxes)

}

+

+// Boxes that disagree about their mode get one warning naming both

+// sides; a stale box is left out of it, and agreement gets none.

+func TestPrintStatusLayoutWarnsWhenBoxesDisagreeAboutTheMode(t *testing.T) {

+ boxes := []statusLayoutBox{

+ {Box: "kass-1", Mode: "nested", Rows: []statusLayoutRow{{Host: "a.example.com", Layout: "nested"}}},

+ {Box: "kass-2", Mode: "none", Rows: []statusLayoutRow{{Host: "b.example.com", Layout: "flat"}}},

+ {Box: "kass-3", Mode: "flat", Rows: []statusLayoutRow{{Host: "c.example.com", Layout: "flat"}}},

+ {Box: "kass-4", Mode: "none", Stale: true, Rows: []statusLayoutRow{{Host: "d.example.com", Layout: "flat"}}},

+ }

+ var buf bytes.Buffer

+ printStatusLayout(&buf, boxes)

+ out := buf.String()

+ want := "WARNING: kass-1 is nested while kass-2 and kass-3 are not: an instance placed or moved onto a box that is not nested starts flat there."

+ if !strings.Contains(out, want) {

+ t.Fatalf("disagreeing boxes not warned about:\n%s", out)

+ }

+ if strings.Contains(out, "kass-4") && strings.Contains(out[strings.Index(out, "WARNING"):], "kass-4") {

+ t.Errorf("a stale box was counted:\n%s", out)

+ }

+ buf.Reset()

+ printStatusLayout(&buf, boxes[:1])

+ if strings.Contains(buf.String(), "WARNING") {

+ t.Errorf("an agreeing fleet warned:\n%s", buf.String())

+ }

+}

diff --git a/system.go b/system.go

index 8253c34ab..2985a966d 100644

--- a/system.go

+++ b/system.go

@@ -47,7 +47,9 @@ func systemVerb(args []string) (verb string, rest []string, component string) {

switch args[0] {

// antivirus is a FAMILY, not a leaf: its own sub-verbs come back

// untouched in rest for runSystemAntivirusVerb to read.

- case "update", "status", "rollback", "apply", "antivirus", "help", "-h", "--help":

+ // layout is a family too: migrate|revert|status stay in rest for

+ // runSystemLayout.

+ case "update", "status", "rollback", "apply", "antivirus", "layout", "help", "-h", "--help":

return args[0], args[1:], component

}

return "", args, component

@@ -66,6 +68,8 @@ func runSystem(args []string) {

runSystemRollback(rest)

case "antivirus":

runSystemAntivirusVerb(rest)

+ case "layout":

+ runSystemLayout(rest)

case "help", "-h", "--help":

systemUsage()

default:

@@ -104,6 +108,8 @@ The platform's own binary (ops account owners and CARLOS_SYSTEM_OPERATORS):

status every box: platform, channel, running build, channel target, last apply outcome

rollback point a channel back at the release before this one

apply (root oneshot behind carlos-system-update.service; not for operators)

+ layout migrate|revert|status: move every instance into its own directory

+ and namespace, or back; see carlos system layout help

Deployment-wide settings (CARLOS_STORE_OPERATORS):

@@ -567,3 +573,146 @@ func printSystemAntivirus(w io.Writer, res apiSystemAntivirusReply) {

mutedf(w, " this alone scans nothing: a member requests scanning for their app,")

mutedf(w, " and an operator grants it per store (carlos store scan grant)")

}

+

+// ---- carlos system layout ----------------------------------------------

+

+// runSystemLayout is `carlos system layout migrate|revert|status`

+// (isolated-exec design §8.2). migrate and revert record an intent through

+// the console's layout door and return at once: the boxes in scope

+// converge on their next serial move, and `carlos status --layout` is the

+// view of that. --force maps to the intent's Force field: an awake

+// instance is stopped for its move now instead of waiting out its grace.

+func runSystemLayout(args []string) {

+ if len(args) == 0 {

+ fmt.Fprint(os.Stderr, systemLayoutUsage)

+ os.Exit(2)

+ }

+ sub, rest := args[0], args[1:]

+ switch sub {

+ case "help", "-h", "--help":

+ fmt.Fprint(os.Stderr, systemLayoutUsage)

+ return

+ case "status", "migrate", "revert":

+ default:

+ fmt.Fprintf(os.Stderr, "system layout: unknown verb %q\n\n%s", sub, systemLayoutUsage)

+ os.Exit(2)

+ }

+ fs := flag.NewFlagSet("system layout "+sub, flag.ExitOnError)

+ consoleFlag := consoleOption(fs, "console origin (default: the only logged-in one)")

+ box := fs.String("box", "", "one box's label; the default is every box")

+ force := fs.Bool("force", false, "stop awake instances for the move now instead of waiting for them to sleep (migrate and revert)")

+ yes := fs.Bool("yes", false, "do not ask")

+ fs.Usage = flagUsage(fs, systemLayoutUsage)

+ fs.Parse(rest)

+ sess := requireDeployAPISession(*consoleFlag)

+ ctx := context.Background()

+

+ if sub == "status" {

+ st, body, err := sess.getJSON(ctx, "/api/cli/system/layout")

+ if err != nil {

+ log.Fatalf("carlos system layout status: %v", err)

+ }

+ if st != http.StatusOK {

+ log.Fatalf("carlos system layout status: %s", systemLayoutDoorError(st, body))

+ }

+ var out systemLayoutResponse

+ if err := json.Unmarshal(body, &out); err != nil {

+ log.Fatalf("carlos system layout status: unreadable response: %v", err)

+ }

+ printSystemLayoutStatus(stdout, out)

+ return

+ }

+

+ mode, what := "nested", "move every instance on %s into its own directory and namespace"

+ if sub == "revert" {

+ mode, what = "flat", "move every isolated instance on %s back to the shared directory"

+ }

+ scope := "every box"

+ if *box != "" {

+ scope = "box " + *box

+ }

+ if !*yes {

+ if !interactiveTerminal() {

+ log.Fatal("no terminal to confirm on; pass --yes")

+ }

+ how := "each one is stopped and moved while it is asleep, or after an hour awake"

+ if *force {

+ how = "awake instances are stopped for the move at once"

+ }

+ if !promptYes(stdout, fmt.Sprintf(what+"? %s [y/N] ", scope, how)) {

+ return

+ }

+ st, body, err := sess.postJSON(ctx, "/api/cli/system/layout", map[string]any{"mode": mode, "box": *box, "force": *force})

+ if err != nil {

+ log.Fatalf("carlos system layout %s: %v", sub, err)

+ }

+ if st != http.StatusOK {

+ log.Fatalf("carlos system layout %s: %s", sub, systemLayoutDoorError(st, body))

+ }

+ fmt.Fprintf(stdout, "%s layout %s for %s; boxes converge on their next serial move. Watch with: carlos status --layout\n",

+ colorFor(stdout).ok("recorded"), mode, scope)

+}

+

+// systemLayoutResponse is the layout door's answer (console

+// apiSystemLayoutResponse). Mode "none" means nothing was ever recorded.

+type systemLayoutResponse struct {

+ Mode string `json:"mode"`

+ Box string `json:"box"`

+ ID string `json:"id"`

+ RequestedAt string `json:"requested_at"`

+ Force bool `json:"force"`

+}

+

+// systemLayoutDoorError is apiErrorFromBody with the one answer this door

+// adds: a bare 404 from a console that predates the door, which the

+// generic membership message would misdescribe.

+func systemLayoutDoorError(st int, body []byte) error {

+ if st == http.StatusNotFound {

+ var e struct {

+ Error string `json:"error"`

+ }

+ if json.Unmarshal(body, &e) != nil || e.Error == "" {

+ return errors.New("not found. this needs an owner of the ops account or an address on CARLOS_SYSTEM_OPERATORS, " +

+ "or this console predates `carlos system layout` (deploy a console built from current main)")

+ }

+ return apiErrorFromBody(st, body)

+}

+

+// printSystemLayoutStatus renders the recorded intent. It is the fleet's

+// INTENT, not what the boxes have done: that is `carlos status --layout`,

+// which the line points at.

+func printSystemLayoutStatus(w io.Writer, out systemLayoutResponse) {

+ if out.Mode == "none" {

+ fmt.Fprintln(w, "no layout intent recorded; every box keeps the flat layout")

+ return

+ }

+ scope := "every box"

+ if out.Box != "" {

+ scope = "box " + out.Box

+ }

+ force := ""

+ if out.Force {

+ force = ", force"

+ }

+ fmt.Fprintf(w, "intent: %s for %s (recorded %s%s)\n", out.Mode, scope, out.RequestedAt, force)

+ fmt.Fprintln(w, "what each box has done: carlos status --layout")

+}

+

+const systemLayoutUsage = `usage: carlos system layout migrate [--box <label>] [--force] [--yes]

+ carlos system layout revert [--box <label>] [--force] [--yes]

+ carlos system layout status

+

+migrate records the intent "nested": every instance the activator manages on

+the boxes in scope is stopped while asleep, moved into its own directory, and

+from its next wake runs in its own mount and pid namespace, where it can read

+its own files and nothing else. revert records "flat" and moves them back; run

+it, and wait for carlos status --layout to show every instance flat, before

+rolling the platform back to a release from before the layout. status prints

+the recorded intent.

+

+--box scopes the request to one box, so a fleet moves one box at a time.

+--force stops an awake instance for its move at once instead of giving it an

+hour to go to sleep first. Ops account owners and CARLOS_SYSTEM_OPERATORS.

+`

diff --git a/system_test.go b/system_test.go

index f910cae9a..d629eb039 100644

--- a/system_test.go

+++ b/system_test.go

@@ -21,6 +21,10 @@ func TestSystemVerbAcceptsTheComponentForm(t *testing.T) {

{[]string{"antivirus", "status"}, "antivirus", "router", 1},

{[]string{"antivirus", "enable", "--console", "x"}, "antivirus", "router", 3},

{[]string{"antivirus"}, "antivirus", "router", 0},

+ // layout is a family too: its sub-verb and flags stay in rest.

+ {[]string{"layout", "migrate", "--box", "x"}, "layout", "router", 3},

+ {[]string{"layout", "revert", "--force", "--yes"}, "layout", "router", 3},

+ {[]string{"layout", "status"}, "layout", "router", 1},

{[]string{}, "", "router", 0},

{[]string{"router"}, "", "router", 0},

}

@@ -245,3 +249,23 @@ func TestPrintSystemAntivirusWarnsOnStaleSignatures(t *testing.T) {

t.Fatalf("stale definitions were not called out:\n%s", b.String())

}

+

+func TestPrintSystemLayoutStatus(t *testing.T) {

+ var b strings.Builder

+ printSystemLayoutStatus(&b, systemLayoutResponse{Mode: "none"})

+ if !strings.Contains(b.String(), "no layout intent recorded") {

+ t.Errorf("none: %q", b.String())

+ }

+ b.Reset()

+ printSystemLayoutStatus(&b, systemLayoutResponse{Mode: "nested", Box: "kass-1", RequestedAt: "2026-10-06T10:00:00Z", Force: true})

+ for _, want := range []string{"intent: nested for box kass-1", "2026-10-06T10:00:00Z, force", "carlos status --layout"} {

+ if !strings.Contains(b.String(), want) {

+ t.Errorf("output lacks %q:\n%s", want, b.String())

+ }

+ b.Reset()

+ printSystemLayoutStatus(&b, systemLayoutResponse{Mode: "flat", RequestedAt: "2026-10-06T10:00:00Z"})

+ if !strings.Contains(b.String(), "intent: flat for every box (recorded 2026-10-06T10:00:00Z)") {

+ t.Errorf("fleet: %q", b.String())

+ }

+}

carlos / platform Public

Isolated exec, plan 5: the console door, `carlos system layout`, the downgrade guards, and nested minting (#126)