amadan
Merge branch 'tenant-iso-default-nested' into main
Isolated exec: nested is the default layout on a ready box; CARLOS_LAYOUT_DEFAULT=flat opts out (#126)
17 files changed,
+728
−45
README.md+20 −0cli.go+31 −13cli_test.go+39 −6docs/site/cli.md+3 −1docs/superpowers/specs/2026-09-11-isolated-exec-design.md+43 −1internal/console/apistatuslayout.go+12 −5internal/console/apistatuslayout_test.go+8 −2internal/console/layoutguard_test.go+21 −0internal/host/agent.go+8 −7internal/host/agent_layout_test.go+267 −0internal/host/layoutleg.go+217 −6internal/status/contract_test.go+1 −1internal/status/snapshot.go+6 −0status_instances.go+18 −0status_instances_test.go+22 −0system.go+8 −1system_test.go+4 −2
CI passed — run details
diff --git a/README.md b/README.md| index 12b34c020..0058c1914 100644 |
| --- a/README.md |
| +++ b/README.md |
| @@ -277,6 +277,26 @@ A release-specific note goes here when a release changes something an |
| operator needs to check or act on before rolling it. Read the note for |
| the release you are about to roll. |
| +**v0.25.0: instances move to the nested layout on their own.** This is |
| +the first release on which an isolated (nested) instance wakes, and nested |
| +is the default. On each box, migration starts unattended on the first |
| +serial move after the release lands and the box's isolation probe passes: |
| +every activator-managed instance that is asleep is moved into its own |
| +directory and namespace at once; one that is awake gets an hour and is |
| +then stopped for its move. An instance whose move is refused for a reason |
| +a tenant could have planted (a file in its new directory, a second name |
| +for its database) is held down automatically, and stays down across agent |
| +restarts until the reason is cleared or the layout is reverted. An |
| +operator who wants control over any of that acts BEFORE rolling: set |
| +`CARLOS_LAYOUT_DEFAULT=flat` in each box's host.env to keep the fleet flat, |
| +or record `carlos system layout migrate --box <label>` to move one box at |
| +a time, or `carlos system layout revert` to opt the fleet out; a recorded |
| +intent always wins over the default. Watch with `carlos status --layout`, |
| +which marks a box `(nested by default)` and says why a box the default |
| +has not reached is still flat. Cut v0.25.0 from a main that contains |
| +7b7d3026 (plan 4's edge openat2 fix); without it nested instances cannot |
| +wake. |
| + |
| **App regions rollback floor.** Once any app holds a region other than the |
| home, do not roll the console binary below the first release that |
| understands app regions: an older console places a no-region create at the |
diff --git a/cli.go b/cli.go| index 0e2277fd7..7e804bbe6 100644 |
| --- a/cli.go |
| +++ b/cli.go |
| @@ -23,6 +23,7 @@ import ( |
| "strings" |
| "time" |
| + "github.com/carlosframework/platform/internal/activator" |
| "github.com/carlosframework/platform/internal/appconfig" |
| "github.com/carlosframework/platform/internal/caps" |
| "github.com/carlosframework/platform/internal/edge" |
| @@ -81,21 +82,33 @@ func refuseAddNested(host string, existing *registry.Route, dataDir string) erro |
| } |
| // addMintsNested reports whether `carlos add` on this box mints the |
| -// nested defaults: the box's layout intent names nested. With no deploy |
| -// store in the environment (a box before its bootstrap, a dev shell) the |
| -// answer is flat, as it always was. An intent that cannot be read is an |
| -// error, not a guess: a flat row on a nested box is the thing this exists |
| -// to prevent, and the operator can pass --socket and --db to say so |
| +// nested defaults: the box's layout intent names nested, or, with no |
| +// intent recorded, the box is already nested by default, which this verb |
| +// can only tell from the registry (nestedByEvidence: an activator-managed |
| +// row already at the nested paths), because the agent's readiness gate is |
| +// not reachable from an operator's shell. With no deploy store in the |
| +// environment (a box before its bootstrap, a dev shell) the registry's |
| +// evidence is all there is. A recorded flat intent is the opt-out and |
| +// wins over the evidence. An intent that cannot be read is an error, not |
| +// a guess: a flat row on a nested box is the thing this exists to |
| +// prevent, and the operator can pass --socket and --db to say so |
| // explicitly. |
| -func addMintsNested(ctx context.Context, store release.Store, boxLabel func() string) (bool, error) { |
| - if store == nil { |
| - return false, nil |
| +func addMintsNested(ctx context.Context, store release.Store, boxLabel func() string, routes []registry.Route, dd string) (bool, error) { |
| + if store != nil { |
| + in, found, err := release.LoadLayoutIntent(ctx, store, boxLabel()) |
| + if err != nil { |
| + return false, fmt.Errorf("could not read the box's layout intent, so the row's paths cannot be chosen: %v. Retry, or pass --socket and --db", err) |
| + } |
| + if found { |
| + return in.Mode == release.LayoutModeNested, nil |
| + } |
| } |
| - in, found, err := release.LoadLayoutIntent(ctx, store, boxLabel()) |
| - if err != nil { |
| - return false, fmt.Errorf("could not read the box's layout intent, so the row's paths cannot be chosen: %v. Retry, or pass --socket and --db", err) |
| + for _, rt := range routes { |
| + if activator.ActivatorManaged(rt) && provision.NestedPaths(dd, rt) { |
| + return true, nil |
| + } |
| } |
| - return found && in.Mode == release.LayoutModeNested, nil |
| + return false, nil |
| } |
| // boxLabelForAdd is the label this box's status publishes under, which is |
| @@ -184,7 +197,12 @@ func runAdd(args []string) { |
| if *kind == "instance" && *backing == "" && *hibernate && *addr == "" && |
| *socket == provision.DefaultSocket(host) && *dbPath == provision.DefaultDBPath(dataDir(), host) { |
| store, _ := deploymentReadSeam() |
| - nested, err := addMintsNested(context.Background(), store, boxLabelForAdd) |
| + var evidence []registry.Route |
| + if r, err := registry.Open(*reg); err == nil { |
| + evidence, _ = r.List() |
| + r.Close() |
| + } |
| + nested, err := addMintsNested(context.Background(), store, boxLabelForAdd, evidence, dataDir()) |
| if err != nil { |
| log.Fatalf("add %s: %v", host, err) |
| } |
diff --git a/cli_test.go b/cli_test.go| index 85ec6fa44..8c6089785 100644 |
| --- a/cli_test.go |
| +++ b/cli_test.go |
| @@ -1406,29 +1406,29 @@ func TestRefuseAddNested(t *testing.T) { |
| func TestAddMintsNestedOnlyUnderANestedIntent(t *testing.T) { |
| ctx := context.Background() |
| label := func() string { return "kass-1" } |
| - if nested, err := addMintsNested(ctx, nil, label); err != nil || nested { |
| + if nested, err := addMintsNested(ctx, nil, label, nil, "/data"); err != nil || nested { |
| t.Fatalf("no store = %v %v, want flat", nested, err) |
| } |
| s := release.NewDirStore(t.TempDir()) |
| - if nested, err := addMintsNested(ctx, s, label); err != nil || nested { |
| + if nested, err := addMintsNested(ctx, s, label, nil, "/data"); err != nil || nested { |
| t.Fatalf("no intent = %v %v, want flat", nested, err) |
| } |
| if _, err := release.AppendLayoutIntent(ctx, s, release.LayoutIntent{Mode: release.LayoutModeNested, Scope: release.LayoutScope{Box: "kass-2"}}); err != nil { |
| t.Fatal(err) |
| } |
| - if nested, err := addMintsNested(ctx, s, label); err != nil || nested { |
| + if nested, err := addMintsNested(ctx, s, label, nil, "/data"); err != nil || nested { |
| t.Fatalf("another box's intent = %v %v, want flat", nested, err) |
| } |
| if _, err := release.AppendLayoutIntent(ctx, s, release.LayoutIntent{Mode: release.LayoutModeNested, Scope: release.LayoutScope{Box: "kass-1"}}); err != nil { |
| t.Fatal(err) |
| } |
| - if nested, err := addMintsNested(ctx, s, label); err != nil || !nested { |
| + if nested, err := addMintsNested(ctx, s, label, nil, "/data"); err != nil || !nested { |
| t.Fatalf("this box's nested intent = %v %v, want nested", nested, err) |
| } |
| if _, err := release.AppendLayoutIntent(ctx, s, release.LayoutIntent{Mode: release.LayoutModeFlat}); err != nil { |
| t.Fatal(err) |
| } |
| - if nested, err := addMintsNested(ctx, s, label); err != nil || nested { |
| + if nested, err := addMintsNested(ctx, s, label, nil, "/data"); err != nil || nested { |
| t.Fatalf("a later fleet-wide flat intent = %v %v, want flat", nested, err) |
| } |
| // Unreadable (a newer schema): refused with the retry named. |
| @@ -1438,7 +1438,40 @@ func TestAddMintsNestedOnlyUnderANestedIntent(t *testing.T) { |
| if _, err := s.Put(ctx, release.LayoutIntentKey(), []byte(`{"schema":2,"requests":[]}`), ""); err != nil { |
| t.Fatal(err) |
| } |
| - if _, err := addMintsNested(ctx, s, label); err == nil || !strings.Contains(err.Error(), "pass --socket and --db") { |
| + if _, err := addMintsNested(ctx, s, label, nil, "/data"); err == nil || !strings.Contains(err.Error(), "pass --socket and --db") { |
| t.Fatalf("unreadable intent = %v, want a refusal naming the flags", err) |
| } |
| } |
| + |
| +// With no intent recorded, `carlos add` mints nested when the box's own |
| +// registry already holds an activator-managed row at the nested paths |
| +// (the box is nested by default and has moved); a recorded flat intent |
| +// wins over that evidence; an unmanaged or external nested-pathed row is |
| +// not evidence. |
| +func TestAddMintsNestedFromTheBoxsOwnRowsWhenNoIntentIsRecorded(t *testing.T) { |
| + ctx := context.Background() |
| + label := func() string { return "kass-1" } |
| + host := "a.example.com" |
| + managedNested := registry.Route{Host: host, Kind: "instance", Hibernate: true, Socket: provision.NestedSocket(host), DBPath: provision.NestedDBPath("/data", host)} |
| + unflaggedNested := registry.Route{Host: host, Kind: "instance", Socket: provision.NestedSocket(host), DBPath: provision.NestedDBPath("/data", host)} |
| + flat := registry.Route{Host: host, Kind: "instance", Hibernate: true, Socket: provision.DefaultSocket(host), DBPath: provision.DefaultDBPath("/data", host)} |
| + if nested, err := addMintsNested(ctx, nil, label, []registry.Route{flat}, "/data"); err != nil || nested { |
| + t.Fatalf("flat rows only = %v %v, want flat", nested, err) |
| + } |
| + if nested, err := addMintsNested(ctx, nil, label, []registry.Route{unflaggedNested}, "/data"); err != nil || nested { |
| + t.Fatalf("an unmanaged nested-pathed row = %v %v, want flat (not evidence)", nested, err) |
| + } |
| + if nested, err := addMintsNested(ctx, nil, label, []registry.Route{flat, managedNested}, "/data"); err != nil || !nested { |
| + t.Fatalf("a managed nested row = %v %v, want nested", nested, err) |
| + } |
| + s := release.NewDirStore(t.TempDir()) |
| + if nested, err := addMintsNested(ctx, s, label, []registry.Route{managedNested}, "/data"); err != nil || !nested { |
| + t.Fatalf("store with no intent, nested evidence = %v %v, want nested", nested, err) |
| + } |
| + if _, err := release.AppendLayoutIntent(ctx, s, release.LayoutIntent{Mode: release.LayoutModeFlat}); err != nil { |
| + t.Fatal(err) |
| + } |
| + if nested, err := addMintsNested(ctx, s, label, []registry.Route{managedNested}, "/data"); err != nil || nested { |
| + t.Fatalf("a recorded flat intent must win over the evidence: %v %v", nested, err) |
| + } |
| +} |
diff --git a/docs/site/cli.md b/docs/site/cli.md| index 05e911548..5872bdcf9 100644 |
| --- a/docs/site/cli.md |
| +++ b/docs/site/cli.md |
| @@ -1753,6 +1753,8 @@ Every instance the activator runs used to live in one shared directory as one us |
| `layout migrate` is refused while any box in scope runs a platform release from before the layout, so a fleet is rolled forward first and migrated second. |
| +You rarely need `migrate` at all. With no intent recorded, a box uses the nested layout by default as soon as its isolation is ready (the probe passes), and moves its instances on its own, with the same hour of grace for an awake one. `carlos status --layout` marks such a box `(nested by default)`, and says why a box the default has not reached is still flat. To keep box-by-box control, record `migrate --box` or `revert` first; a recorded intent always wins. A deployment that is not ready for this at all puts `CARLOS_LAYOUT_DEFAULT=flat` in each box's host.env. |
| + |
| `layout revert` moves them back. The order for rolling the platform back to a release from before the layout is: `layout revert`, wait until `carlos status --layout` shows every instance `flat`, then `system rollback`. The console will not move a channel pointer onto such a release while any box reports otherwise, and a revert is refused while a maintenance worker is attached to a box in scope. `layout status` prints the recorded intent; what each box has done is `carlos status --layout`. |
| ```sh |
| @@ -1765,7 +1767,7 @@ carlos system layout status |
| Ops account owners and `CARLOS_SYSTEM_OPERATORS`, like `update` and `rollback`. |
| -A box is put on canary with `CARLOS_SYSTEM_CHANNEL=canary` in its host.env. |
| +A box is put on canary with `CARLOS_SYSTEM_CHANNEL=canary` in its host.env, and opts out of the nested instance layout by default with `CARLOS_LAYOUT_DEFAULT=flat` there. |
| Boxes that have not reported a platform yet count as stable. `carlos system |
| apply` is the root oneshot behind `carlos-system-update.service`; it is not |
| for operators. The console is an app — update it with `carlos deploy --app |
diff --git a/docs/superpowers/specs/2026-09-11-isolated-exec-design.md b/docs/superpowers/specs/2026-09-11-isolated-exec-design.md| index b1eba239e..f1a1cb222 100644 |
| --- a/docs/superpowers/specs/2026-09-11-isolated-exec-design.md |
| +++ b/docs/superpowers/specs/2026-09-11-isolated-exec-design.md |
| @@ -1,6 +1,6 @@ |
| # Isolated exec — every exec-backed instance in its own namespace |
| -**Date:** 2026-09-11 · **Status:** revision 16; codex verdict after |
| +**Date:** 2026-09-11 · **Status:** revision 17; codex verdict after |
| round 11 is **accept with corrections**, and the one correction is |
| applied here (54 findings over 11 rounds, all accepted; §13); a |
| whole-branch final review (plan 2) found six further items, fixed in |
| @@ -1769,6 +1769,47 @@ fails only on a line that names the property that is missing. |
| there, not even for one pass. `DefaultDBPath`/`DefaultSocket` stay |
| flat for every other caller until the next release. |
| +**Nested by default (revision 17, Paul's decision 2026-10-06).** From |
| +the release that carries plan 5 (v0.25.0), nested is the default, not an |
| +opt-in. When `control/layout.json` is absent, the effective intent on a |
| +box is `nested` **if and only if the box is ready**: the deployment's |
| +default is nested (`CARLOS_LAYOUT_DEFAULT` in host.env is unset or |
| +`nested`; `flat` is the per-deployment opt-out, for a self-hoster whose |
| +boxes are not ready), the running build is at or above `LayoutFloor` |
| +when it can be placed (a build that evaluates the rule carries the pass |
| +by construction, so an unplaceable dev build passes and a mis-tagged |
| +release below the floor does not), and the nested gate of §8.5 answers: |
| +`instancefs.SelfCheck`, the three `nosymfollow` mounts, and the control |
| +service's probe verdict. A box that is not ready behaves exactly as |
| +before the default (flat, mints flat) and says why in its status |
| +report (`layout.default_blocked`); a box that is nested by default says |
| +so (`layout.default`), so `carlos status --layout` can tell a box that |
| +was migrated on purpose from one that moved on its own. |
| + |
| +A recorded intent always wins: an explicit `flat` is the opt-out, an |
| +explicit `nested` behaves as today, and `--box` scoping is how an |
| +operator keeps box-by-box control. The default never forces: the awake |
| +grace, the holds and the quarantine of plan 4 apply as they do under a |
| +recorded intent. The console's guards need no new rule: the downgrade |
| +guard and the migrate guard judge what each box REPORTS (its rows and |
| +its running release), so a box that is nested by default is treated |
| +exactly like one under an explicit intent wherever it reports nested |
| +rows. `carlos add` cannot reach the gate from an operator's shell, so |
| +with no intent recorded it mints nested only when the box's registry |
| +already holds an activator-managed row at the nested paths; a fresh |
| +box's first hand-added row is minted flat and moved by the pass on the |
| +next serial move. |
| + |
| +Rollout, restated: the flagship cuts v0.25.0 with this in, rolls it, and |
| +each box migrates on its own once its probe reports ready. An operator |
| +who wants box-by-box control records `carlos system layout migrate |
| +--box` (or `revert`) first; the recorded intent then governs. |
| + |
| +Why a default and not a verb: the §1.2 list is what every flat tenant |
| +can reach, and a fleet that stays flat because nobody ran a verb is a |
| +fleet that stays exposed by omission. The gate is what makes the default |
| +safe to have: a box that cannot isolate does nothing, loudly. |
| + |
| **The roll script and the planner.** `update-platform-binary.sh:52-70` |
| (pinfra) enables units by name; it gains `carlos-instance-control.socket`, |
| the three mount units and the probe, before it restarts the edge, and |
| @@ -2201,5 +2242,6 @@ rollback runs the old binary). |
| - R12 (final review, plan 2): cgroup-ro targets the edge's delegated cgroup.procs; DAC precedes the mount flag so a root-owned file answers EACCES. |
| - R13 (task 4 review, plan 3): after a stop the activator requested, a unit reporting a foreign or absent invocation proves the old invocation over without the cgroup gate (§6.5); the observer never stops an invocation it did not start once its own stop was requested. |
| - R14 (plan 4, 2026-10-04; recorded in revision 16): the per-host layout view is `carlos status --layout`, beside `--instances` and under the same operator gate, not `carlos instances --layout`: the instances verb lists the local registry, and the layout report lives in each box's private status snapshot, which only the console reads (§8.4). |
| +- R17 (plan 5 follow-up, Paul's decision 2026-10-06; revision 17): nested is the default when no intent is recorded, gated on the box's readiness; `CARLOS_LAYOUT_DEFAULT=flat` is the deployment opt-out and a recorded `flat` intent the fleet opt-out (§8.6, "Nested by default"). |
| - R15 (first roll, 2026-09-12; fixed 2026-09-25): the probe on dev's boxes answered `ready: false` twice, and the review of the fix found a third gap. (a) `RestrictSUIDSGID=yes` makes `openat2` fail with ENOSYS on systemd 257, because systemd cannot read the mode inside `open_how` and so blocks the whole call; every `instancefs` open in the namespace is an `openat2`. The line is dropped from all four templates. What it guarded is covered another way: with no capabilities an instance can only make a set-id file owned by `carlos`, which gains nothing for another `carlos` process; `NoNewPrivileges=yes` voids it inside the namespace; and no other uid can reach one (the tenant trees are `nosuid`, and `PrivateTmp=`'s `/tmp` and `/var/tmp` sit under a root 0700 directory, `/var/tmp` being on the root filesystem and not `nosuid`). (b) `control-unreachable` checked `Lstat`, but `InaccessiblePaths=` leaves a mode 000 root-owned node at the path. The check is now what a tenant could do, a listing of the directory and a dial of the socket, and passes only when both are refused for permission or absence; a refused connection or a timeout proves the path reachable and fails. (c) `PrivateDevices=` bind-mounts the host's `/dev/shm`, so every instance, all uid `carlos`, shared one; each unit now gets its own tmpfs there, and the probe's `shm-private` check plants a file in the host's `/dev/shm` (the control service's tmpfiles fixture) and fails if it is visible. POSIX message queues and abstract unix sockets are still box-wide; they are a follow-up (§11). (d) 2026-10-04, plan 4: `carlos-edge.service` carried the same line, and the agent runs there: its nested `instancefs` opens and every rename of the layout pass are `openat2` calls, so the first `layout migrate` would have met ENOSYS on its first host. The line is dropped from the edge unit too. The argument in (a) carries: the edge's one capability, `CAP_NET_BIND_SERVICE`, does not bear on set-id files (that takes `CAP_CHOWN`, `CAP_FSETID` or `CAP_SETUID`), so anything the agent could mark set-id is owned by `carlos`, and the agent and its flat children run under `NoNewPrivileges=`. An `openat` fallback in `instancefs` was considered and rejected: it would be a second resolution path, exercised in production only when the hardening had regressed, and it would route around the regression rather than report it. Instead the agent runs `instancefs.SelfCheck` in its own unit at start, and the nested-wake gate (§8.5) asks it first, so a blocked `openat2` refuses nested wakes and the layout pass with the directive named. |
| - R16 (plan 4 review, 2026-10-04): three gaps in the pass while flat tenants still share the agent's uid. (a) A tenant could make its own future directory under `/data/.instances` ahead of the pass and hard-link a sibling's database or `litestream.env` into it (its own uid's files, so `protected_hardlinks` allows it); `EnsureDirs` accepted the directory and the destination check looked only at the planned names. Now a migration refuses an instance directory holding any entry the plan does not name, refuses any file about to move that has more than one name (the shadow directory walked), and checks both again after the renames and before the row flips; a failure there keeps the journal at `files` and the host fenced. (b) Deferring an awake host (plan 4 amendment, "when it is asleep") was unbounded, so a tenant that never idled kept raw-path access to its nested siblings for good. The deferral is bounded: an hour of grace (`layout.DefaultAwakeGrace`), after which the pass stops the host as §8.2 step 2 always allowed and moves it; the serial leg re-runs on its own ten minutes after any deferral so the bound holds on a quiet fleet; the intent gains `force` for an operator who wants no grace; and `carlos status --layout` warns on every box with nested rows beside flat ones, because that box is not isolated yet. (c) Files moved aside at a destination (`<name>.aside-<unix>`) were kept inside the instance directory, readable by the tenant that gets the directory; they are kept under `/data/layout/aside` now, which no instance namespace contains. Still open: a flat tenant that is running while another host moves can plant between the check and the rename; the check is against what was planted before the pass arrived, and the closure is the bound in (b). (d) Re-check: (a)'s refusals were themselves escapable: a tenant could keep its own move refused with one syscall (`ln /data/B.db /data/.x`, or a file in its own future directory), and because the pass lifted its fence on a refusal the host woke flat again on its next request, for as long as it kept the refusal alive. The rule is now fail-closed on the way to nested: a flat host that has been stopped for its move and is then refused stays fenced and down (the pass keeps the fence, as it already did after a failed handover), status reports it `fenced` with the refusal (`held down after a refused move to nested`), every later pass retries from the kept fence and releases the host once it has moved, and a flat intent lifts the fence. A nested host refused its revert is not held: it is already isolated. The mixed-layout warning counts `foreign` and `external` rows beside `flat` ones, since all three run as uid carlos with no namespace; and the second-name refusal reports the inode and the `find <data> -xdev -samefile <path>` that lists the other names. (e) Landing follow-ups: a planted ENTRY in a host's future directory is no longer a refusal (one `touch` by a sibling would have held the host down until an operator acted, and the hold outlived the sibling's own migration); it is renamed into `/data/layout/aside/<host>/` with a random, no-replace name and the move goes on, logged and shown in status as a note. The hold stays for a second name (`nlink > 1`, no safe automatic fix) and for a symlink at the instance directory. The hold is persisted as `/data/layout/<host>.hold` and restored at agent start before anything wakes, by re-deriving it from the files: the marker is a hint to re-check, not the authority, because every file under `/data` is uid carlos's and a flat tenant can unlink it; a deleted marker therefore costs one grace after one restart, after which the next pass meets the same refusal and holds again, and a marker whose reason is gone is dropped rather than honoured. The serial leg re-runs on its own after a held result, as it does after a deferral, so a cleared reason is acted on within ten minutes on a quiet fleet. |
diff --git a/internal/console/apistatuslayout.go b/internal/console/apistatuslayout.go| index 8623edc1c..9471f25e4 100644 |
| --- a/internal/console/apistatuslayout.go |
| +++ b/internal/console/apistatuslayout.go |
| @@ -45,6 +45,11 @@ type apiStatusLayoutBox struct { |
| // directory: the nested rows are not isolated until none remains on |
| // the box. Additive; an old CLI ignores it. |
| Mixed bool `json:"mixed,omitempty"` |
| + // Default is set when the box is nested by the deployment's default |
| + // rather than a recorded intent; DefaultBlocked is why the default is |
| + // not in force on a box whose mode is none (design §8.6). Additive. |
| + Default bool `json:"default,omitempty"` |
| + DefaultBlocked string `json:"default_blocked,omitempty"` |
| } |
| // layoutMixed is the Mixed rule: a row that runs as the agent's uid with |
| @@ -109,11 +114,13 @@ func (s *Server) handleAPIStatusLayout(w http.ResponseWriter, r *http.Request) { |
| } |
| sort.Slice(rows, func(i, j int) bool { return rows[i].Host < rows[j].Host }) |
| out = append(out, apiStatusLayoutBox{ |
| - Box: b.Label, |
| - Mode: b.Metrics.Layout.Mode, |
| - Stale: status.BoxStale(b, now, status.SnapshotPeriod), |
| - Rows: rows, |
| - Mixed: layoutMixed(b.Metrics.Layout.Mode, rows), |
| + Box: b.Label, |
| + Mode: b.Metrics.Layout.Mode, |
| + Stale: status.BoxStale(b, now, status.SnapshotPeriod), |
| + Rows: rows, |
| + Mixed: layoutMixed(b.Metrics.Layout.Mode, rows), |
| + Default: b.Metrics.Layout.Default, |
| + DefaultBlocked: b.Metrics.Layout.DefaultBlocked, |
| }) |
| } |
| sort.Slice(out, func(i, j int) bool { return out[i].Box < out[j].Box }) |
diff --git a/internal/console/apistatuslayout_test.go b/internal/console/apistatuslayout_test.go| index 0d2f383bc..c0f81efb9 100644 |
| --- a/internal/console/apistatuslayout_test.go |
| +++ b/internal/console/apistatuslayout_test.go |
| @@ -75,7 +75,7 @@ func TestAPIStatusLayoutListsBoxesRowsAndStale(t *testing.T) { |
| now := time.Now() |
| seedLayoutBox(t, fx.rel, "fresh-box", now, &status.LayoutReport{ |
| - Mode: "nested", |
| + Mode: "nested", Default: true, |
| Rows: map[string]status.RowLayout{ |
| "a.example.com": {Layout: "nested"}, |
| "b.example.com": {Layout: "foreign", Reason: "db and socket are neither default"}, |
| @@ -83,7 +83,7 @@ func TestAPIStatusLayoutListsBoxesRowsAndStale(t *testing.T) { |
| }) |
| // Stale: past staleFactor(3)*SnapshotPeriod(5m) = 15 minutes dark. |
| seedLayoutBox(t, fx.rel, "stale-box", now.Add(-20*time.Minute), &status.LayoutReport{ |
| - Mode: "flat", |
| + Mode: "flat", DefaultBlocked: "isolation not ready: probe fixture absent", |
| Rows: map[string]status.RowLayout{"c.example.com": {Layout: "flat"}}, |
| }) |
| @@ -125,6 +125,12 @@ func TestAPIStatusLayoutListsBoxesRowsAndStale(t *testing.T) { |
| } |
| fresh := byLabel["fresh-box"] |
| + if !fresh.Default || fresh.DefaultBlocked != "" { |
| + t.Errorf("fresh-box = %+v, want Default carried through from the snapshot", fresh) |
| + } |
| + if stale := byLabel["stale-box"]; stale.Default || stale.DefaultBlocked != "isolation not ready: probe fixture absent" { |
| + t.Errorf("stale-box = %+v, want DefaultBlocked carried through", stale) |
| + } |
| if !fresh.Mixed { |
| t.Errorf("fresh-box has a nested row beside a foreign one (uid carlos, no namespace) and is not marked mixed: %+v", fresh) |
| } |
diff --git a/internal/console/layoutguard_test.go b/internal/console/layoutguard_test.go| index 57467f163..2154ddcb3 100644 |
| --- a/internal/console/layoutguard_test.go |
| +++ b/internal/console/layoutguard_test.go |
| @@ -213,3 +213,24 @@ func TestTodaysFleetWithNoLayoutFieldKeepsRollbackAndUpdate(t *testing.T) { |
| t.Fatalf("a sha box with no layout field: %q", got) |
| } |
| } |
| + |
| +// A box that is nested by default (no recorded intent) is judged exactly |
| +// like one under an explicit intent: its nested rows block a pre-floor |
| +// pointer move, and a box the default has not reached (flat rows, a |
| +// blocked reason) does not. The guards never read the intent, so the |
| +// default needs no rule of its own; this pins that. |
| +func TestLayoutGuardsTreatNestedByDefaultLikeAnIntent(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + now := time.Now() |
| + seedLayoutBox(t, fx.rel, "ie", now, &status.LayoutReport{Mode: "nested", Default: true, |
| + Rows: map[string]status.RowLayout{"a.example.com": {Layout: "nested"}}}) |
| + if got := fx.srv.layoutDowngradeBlocked(ctx, "v0.19.0"); !strings.Contains(got, "ie: a.example.com is nested") { |
| + t.Fatalf("nested by default did not block: %q", got) |
| + } |
| + seedLayoutBox(t, fx.rel, "ie", now, &status.LayoutReport{Mode: "none", DefaultBlocked: "isolation not ready: probe fixture absent", |
| + Rows: map[string]status.RowLayout{"a.example.com": {Layout: "flat"}}}) |
| + if got := fx.srv.layoutDowngradeBlocked(ctx, "v0.19.0"); got != "" { |
| + t.Fatalf("a box the default has not reached blocked: %q", got) |
| + } |
| +} |
diff --git a/internal/host/agent.go b/internal/host/agent.go| index b5b313e88..c74c19427 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -93,6 +93,10 @@ |
| // datacenter. |
| // - CARLOS_SYSTEM_CHANNEL — platform channel this box follows for |
| // self-update, canary|stable (default stable). |
| +// - CARLOS_LAYOUT_DEFAULT — the instance layout this box converges to |
| +// when no layout intent is recorded, nested|flat (default nested; |
| +// isolated-exec design §8.6, "Nested by default"). Nested applies only |
| +// on a box whose isolation is ready; a recorded intent always wins. |
| // - STATUS_HOST / STATUS_PUBLIC — NOT read anywhere in this file. They |
| // gate whether ops/tofu fronts the bucket's published status/ prefix |
| // with a public hostname; recorded here only so the fleet status |
| @@ -1107,7 +1111,9 @@ func reconcileSerialized(ctx context.Context, s release.Store, reg *registry.Reg |
| // through, because the intent is read from the store this pass has in |
| // hand and scoped to the label this box publishes under. |
| if cfg.Layout == nil { |
| - cfg.Layout = mintLayoutIntent |
| + cfg.Layout = func(ctx context.Context, s release.Store) (string, error) { |
| + return mintLayoutIntentWith(ctx, s, reg.List) |
| + } |
| } |
| return errors.Join( |
| provision.Reconcile(ctx, s, reg, cfg), |
| @@ -2177,12 +2183,7 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service, sd *schedu |
| // The layout leg: this box's label is the one its status publishes |
| // under, which is the label an operator scopes a layout request to. |
| layoutBox := metricsBoxLabel(os.Getenv, readMachineID()) |
| - layoutPasses := &layoutLeg{} |
| - layoutPasses.run = func(ctx context.Context) { |
| - if runLayoutLeg(ctx, store, layoutBox) { |
| - layoutPasses.retryLater(ctx, awakeRetry) |
| - } |
| - } |
| + layoutPasses := newLayoutLeg(ctx, func(ctx context.Context) bool { return runLayoutLeg(ctx, store, layoutBox) }) |
| d.layoutLeg = layoutPasses.kick |
| // The platform self-update leg (spec §4): the sandboxed half, which only |
| // ever fetches into /data and writes a request. It lives here rather than |
diff --git a/internal/host/agent_layout_test.go b/internal/host/agent_layout_test.go| index c75caa9ae..958bfe704 100644 |
| --- a/internal/host/agent_layout_test.go |
| +++ b/internal/host/agent_layout_test.go |
| @@ -6,6 +6,7 @@ import ( |
| "os" |
| "path/filepath" |
| "reflect" |
| + "strings" |
| "sync" |
| "sync/atomic" |
| "testing" |
| @@ -27,6 +28,8 @@ func resetLayoutGlobals(t *testing.T) { |
| armedLayoutPass.Store(nil) |
| layoutReload.Store(nil) |
| layoutIntentSeen.Store(nil) |
| + layoutDefaultGate.Store(nil) |
| + layoutDefaultSeen.Store(nil) |
| } |
| clear() |
| t.Cleanup(clear) |
| @@ -275,6 +278,10 @@ func TestStartActivatorRecoversAfterTheSweepAndBeforeStart(t *testing.T) { |
| func TestStartupLayoutIntent(t *testing.T) { |
| ctx := context.Background() |
| resetLayoutGlobals(t) |
| + // Recorded intents are what this test is about; opt out of the nested |
| + // default so "no object" reads as none (TestStartupAndMintReadTheDefault |
| + // covers the default). |
| + t.Setenv(layoutDefaultEnv, "flat") |
| if got := startupLayoutIntent(ctx, nil, errors.New("no bucket"), "kass-1"); got != layout.IntentUnreadable { |
| t.Fatalf("no store: %q", got) |
| } |
| @@ -775,3 +782,263 @@ func TestSyncExecSerializedFencedClaimHost(t *testing.T) { |
| t.Fatalf("unfenced host lost its host-write option: %v", err) |
| } |
| } |
| + |
| +// armDefaultGate makes the default's readiness answer what the test says. |
| +func armDefaultGate(t *testing.T, err error) { |
| + t.Helper() |
| + gate := func() error { return err } |
| + layoutDefaultGate.Store(&gate) |
| +} |
| + |
| +// defaultLayoutIntent (design §8.6, "Nested by default") is nested only |
| +// when every readiness rule holds; each rule that fails names itself. |
| +func TestDefaultLayoutIntentRules(t *testing.T) { |
| + resetLayoutGlobals(t) |
| + prev := layoutBuild |
| + t.Cleanup(func() { layoutBuild = prev }) |
| + layoutBuild = func() string { return "v0.25.0" } |
| + |
| + // No activator armed: not ready, says so. |
| + t.Setenv(layoutDefaultEnv, "") |
| + if mode, reason := defaultLayoutIntent(); mode != "" || !strings.Contains(reason, "no activator") { |
| + t.Fatalf("no gate = %q %q", mode, reason) |
| + } |
| + // Gate refusing: not ready, with the gate's reason. |
| + armDefaultGate(t, errors.New("instance control socket not listening")) |
| + if mode, reason := defaultLayoutIntent(); mode != "" || !strings.Contains(reason, "isolation not ready: instance control socket not listening") { |
| + t.Fatalf("gate refusing = %q %q", mode, reason) |
| + } |
| + // Ready: nested by default. |
| + armDefaultGate(t, nil) |
| + if mode, reason := defaultLayoutIntent(); mode != release.LayoutModeNested || reason != "" { |
| + t.Fatalf("ready = %q %q, want nested", mode, reason) |
| + } |
| + // The deployment opt-out wins over readiness. |
| + t.Setenv(layoutDefaultEnv, "flat") |
| + if mode, reason := defaultLayoutIntent(); mode != "" || reason != "" { |
| + t.Fatalf("CARLOS_LAYOUT_DEFAULT=flat = %q %q, want none with no reason", mode, reason) |
| + } |
| + t.Setenv(layoutDefaultEnv, " Nested ") |
| + if mode, _ := defaultLayoutIntent(); mode != release.LayoutModeNested { |
| + t.Fatalf("CARLOS_LAYOUT_DEFAULT=' Nested ' = %q, want nested (folded)", mode) |
| + } |
| + // A typo stays flat and says so. |
| + t.Setenv(layoutDefaultEnv, "nestd") |
| + if mode, reason := defaultLayoutIntent(); mode != "" || !strings.Contains(reason, "CARLOS_LAYOUT_DEFAULT=nestd is not nested or flat") { |
| + t.Fatalf("typo = %q %q", mode, reason) |
| + } |
| + // A release below the floor is not ready, however the gate answers; |
| + // an unplaceable build (this code, running) passes the floor. |
| + t.Setenv(layoutDefaultEnv, "nested") |
| + layoutBuild = func() string { return "v0.24.2" } |
| + if mode, reason := defaultLayoutIntent(); mode != "" || !strings.Contains(reason, "runs v0.24.2, below v0.25.0") { |
| + t.Fatalf("below floor = %q %q", mode, reason) |
| + } |
| + layoutBuild = func() string { return "abc1234" } |
| + if mode, _ := defaultLayoutIntent(); mode != release.LayoutModeNested { |
| + t.Fatalf("dev build = %q, want nested", mode) |
| + } |
| +} |
| + |
| +// The serial leg under no recorded intent: a ready box converges nested, |
| +// never forced, and status says nested by default; a box that is not ready |
| +// converges nothing and status says why; a recorded flat intent is the |
| +// opt-out and wins; a recorded nested intent is not "by default". |
| +func TestLayoutLegUsesTheDefaultWhenNoIntentIsRecorded(t *testing.T) { |
| + resetLayoutGlobals(t) |
| + armedLayoutPass.Store(&layout.Pass{DataDir: t.TempDir(), SocketDir: t.TempDir()}) |
| + t.Setenv(layoutDefaultEnv, "") |
| + prev := layoutBuild |
| + t.Cleanup(func() { layoutBuild = prev }) |
| + layoutBuild = func() string { return "v0.25.0" } |
| + s := release.NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + type call struct { |
| + mode layout.Mode |
| + force bool |
| + } |
| + var calls []call |
| + converge := func(_ context.Context, mode layout.Mode, force bool) []layout.Result { |
| + calls = append(calls, call{mode, force}) |
| + return nil |
| + } |
| + |
| + armDefaultGate(t, errors.New("probe: pid1 fail")) |
| + layoutLegWith(ctx, s, "kass-1", converge) |
| + if len(calls) != 0 { |
| + t.Fatalf("converged while not ready: %v", calls) |
| + } |
| + if got := layoutStatusReport(nil); got.Mode != "none" || got.Default || !strings.Contains(got.DefaultBlocked, "isolation not ready: probe: pid1 fail") { |
| + t.Fatalf("report while not ready = %+v", got) |
| + } |
| + |
| + armDefaultGate(t, nil) |
| + layoutLegWith(ctx, s, "kass-1", converge) |
| + if !reflect.DeepEqual(calls, []call{{layout.ModeNested, false}}) { |
| + t.Fatalf("calls = %v, want one unforced nested converge", calls) |
| + } |
| + if got := layoutStatusReport(nil); got.Mode != "nested" || !got.Default || got.DefaultBlocked != "" { |
| + t.Fatalf("report when nested by default = %+v", got) |
| + } |
| + |
| + // The opt-out: a recorded flat intent wins over a ready box. |
| + appendIntent(t, s, release.LayoutModeFlat, "") |
| + layoutLegWith(ctx, s, "kass-1", converge) |
| + if len(calls) != 2 || calls[1].mode != layout.ModeFlat { |
| + t.Fatalf("calls after a flat intent = %v, want a flat converge", calls) |
| + } |
| + if got := layoutStatusReport(nil); got.Mode != "flat" || got.Default || got.DefaultBlocked != "" { |
| + t.Fatalf("report under a recorded flat intent = %+v", got) |
| + } |
| + |
| + // A recorded nested intent is nested on purpose, not by default. |
| + appendIntent(t, s, release.LayoutModeNested, "kass-1") |
| + layoutLegWith(ctx, s, "kass-1", converge) |
| + if got := layoutStatusReport(nil); got.Mode != "nested" || got.Default { |
| + t.Fatalf("report under a recorded nested intent = %+v, want nested and not default", got) |
| + } |
| +} |
| + |
| +// Startup recovery and the reconciler's mint read the same default: with |
| +// no intent and a ready box both answer nested; with no intent and a box |
| +// that is not ready both answer none; CARLOS_LAYOUT_DEFAULT=flat answers |
| +// none on a ready box. |
| +func TestStartupAndMintReadTheDefault(t *testing.T) { |
| + resetLayoutGlobals(t) |
| + t.Setenv(layoutDefaultEnv, "") |
| + prev := layoutBuild |
| + t.Cleanup(func() { layoutBuild = prev }) |
| + layoutBuild = func() string { return "v0.25.0" } |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + |
| + // Startup never asks the gate: under the nested default it runs |
| + // recovery fail-closed (IntentUnreadable keeps holds and journals), |
| + // whatever the gate would say, and status says why. |
| + armDefaultGate(t, errors.New("not yet run")) |
| + if got := startupLayoutIntent(ctx, s, nil, "kass-1"); got != layout.IntentUnreadable { |
| + t.Fatalf("startup, gate not ready = %q, want unreadable (holds kept)", got) |
| + } |
| + if d := lastLayoutDefault(); d.Applied || !strings.Contains(d.Blocked, "agent starting: holds and journals kept") { |
| + t.Fatalf("default state at start = %+v", d) |
| + } |
| + armDefaultGate(t, nil) |
| + if got := startupLayoutIntent(ctx, s, nil, "kass-1"); got != layout.IntentUnreadable { |
| + t.Fatalf("startup, gate ready = %q, want unreadable: startup does not trust the gate", got) |
| + } |
| + // The mint asks the gate; not ready is flat... |
| + armDefaultGate(t, errors.New("not yet run")) |
| + if mode, err := mintLayoutIntent(ctx, s); err != nil || mode != "" { |
| + t.Fatalf("mint, not ready = %q %v, want none", mode, err) |
| + } |
| + // ...unless the box already holds an activator-managed nested row, |
| + // which is proof it has moved: a fresh row beside it is minted nested |
| + // before the gate's first answer. A nested row that is not managed, |
| + // or flat rows, are not that proof. |
| + t.Setenv("CARLOS_DATA_DIR", "/data") |
| + nestedManaged := registry.Route{Host: "a.example.com", Kind: "instance", Hibernate: true, DBPath: provision.NestedDBPath("/data", "a.example.com"), Socket: provision.NestedSocket("a.example.com")} |
| + nestedUnflagged := registry.Route{Host: "u.example.com", Kind: "instance", DBPath: provision.NestedDBPath("/data", "u.example.com"), Socket: provision.NestedSocket("u.example.com")} |
| + flat := registry.Route{Host: "f.example.com", Kind: "instance", Hibernate: true, DBPath: provision.DefaultDBPath("/data", "f.example.com"), Socket: provision.DefaultSocket("f.example.com")} |
| + rows := func(rs ...registry.Route) func() ([]registry.Route, error) { |
| + return func() ([]registry.Route, error) { return rs, nil } |
| + } |
| + if mode, err := mintLayoutIntentWith(ctx, s, rows(flat, nestedUnflagged)); err != nil || mode != "" { |
| + t.Fatalf("mint, not ready, no managed nested row = %q %v, want none", mode, err) |
| + } |
| + if mode, err := mintLayoutIntentWith(ctx, s, rows(flat, nestedManaged)); err != nil || mode != release.LayoutModeNested { |
| + t.Fatalf("mint, not ready, a managed nested row = %q %v, want nested", mode, err) |
| + } |
| + if _, err := mintLayoutIntentWith(ctx, s, func() ([]registry.Route, error) { return nil, errors.New("registry locked") }); err == nil { |
| + t.Fatal("a registry that cannot be listed must defer the mint, not guess") |
| + } |
| + armDefaultGate(t, nil) |
| + if mode, err := mintLayoutIntent(ctx, s); err != nil || mode != release.LayoutModeNested { |
| + t.Fatalf("mint, ready = %q %v, want nested", mode, err) |
| + } |
| + // The opt-out: startup drops holds as a recorded flat would, and the |
| + // mint is flat even beside a nested row. |
| + t.Setenv(layoutDefaultEnv, "flat") |
| + if got := startupLayoutIntent(ctx, s, nil, "kass-1"); got != "" { |
| + t.Fatalf("startup, opt-out = %q, want none", got) |
| + } |
| + if mode, err := mintLayoutIntentWith(ctx, s, rows(nestedManaged)); err != nil || mode != "" { |
| + t.Fatalf("mint, opt-out = %q %v, want none", mode, err) |
| + } |
| + // A recorded intent still wins over the opt-out. |
| + appendIntent(t, s, release.LayoutModeNested, "") |
| + if mode, err := mintLayoutIntent(ctx, s); err != nil || mode != release.LayoutModeNested { |
| + t.Fatalf("mint under a recorded nested intent with the opt-out set = %q %v, want nested", mode, err) |
| + } |
| +} |
| + |
| +// newLayoutPass arms the default's gate with the same nestedOK the pass |
| +// refuses nested work with. |
| +func TestNewLayoutPassArmsTheDefaultGate(t *testing.T) { |
| + resetLayoutGlobals(t) |
| + want := errors.New("the pass's gate") |
| + newLayoutPass(nil, nil, t.TempDir(), func() error { return want }, true) |
| + gate := layoutDefaultGate.Load() |
| + if gate == nil || !errors.Is((*gate)(), want) { |
| + t.Fatal("the default's gate is not the pass's nestedOK") |
| + } |
| +} |
| + |
| +// newLayoutLeg kicks the leg once at build when a pass is armed, so the |
| +// first evaluation with the gate's real answer never waits on a console |
| +// bump; with no pass armed there is nothing to evaluate and no kick. |
| +func TestNewLayoutLegKicksOnceWhenAPassIsArmed(t *testing.T) { |
| + resetLayoutGlobals(t) |
| + var runs atomic.Int32 |
| + run := func(context.Context) bool { runs.Add(1); return false } |
| + |
| + leg := newLayoutLeg(context.Background(), run) |
| + leg.wait() |
| + if runs.Load() != 0 { |
| + t.Fatalf("runs with no pass armed = %d, want 0", runs.Load()) |
| + } |
| + |
| + armedLayoutPass.Store(&layout.Pass{DataDir: t.TempDir(), SocketDir: t.TempDir()}) |
| + leg = newLayoutLeg(context.Background(), run) |
| + leg.wait() |
| + if runs.Load() != 1 { |
| + t.Fatalf("runs with a pass armed = %d, want exactly one startup kick", runs.Load()) |
| + } |
| + // Later kicks are the serial loop's, as before. |
| + leg.kick(context.Background()) |
| + leg.wait() |
| + if runs.Load() != 2 { |
| + t.Fatalf("runs after a serial kick = %d, want 2", runs.Load()) |
| + } |
| +} |
| + |
| +// A run under the nested default that the gate refuses asks for the leg's |
| +// retry, so a quiet fleet re-asks the gate on its own; the opt-out and a |
| +// typo do not, because nothing they wait on will change. |
| +func TestLayoutLegRetriesWhileTheDefaultIsBlocked(t *testing.T) { |
| + resetLayoutGlobals(t) |
| + armedLayoutPass.Store(&layout.Pass{DataDir: t.TempDir(), SocketDir: t.TempDir()}) |
| + prev := layoutBuild |
| + t.Cleanup(func() { layoutBuild = prev }) |
| + layoutBuild = func() string { return "v0.25.0" } |
| + s := release.NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + converge := func(context.Context, layout.Mode, bool) []layout.Result { return nil } |
| + |
| + t.Setenv(layoutDefaultEnv, "") |
| + armDefaultGate(t, errors.New("not yet run")) |
| + if !layoutLegWith(ctx, s, "kass-1", converge) { |
| + t.Fatal("a gate that is not ready must ask for a retry") |
| + } |
| + armDefaultGate(t, nil) |
| + if layoutLegWith(ctx, s, "kass-1", converge) { |
| + t.Fatal("a ready box with nothing deferred must not ask for a retry") |
| + } |
| + t.Setenv(layoutDefaultEnv, "flat") |
| + if layoutLegWith(ctx, s, "kass-1", converge) { |
| + t.Fatal("the opt-out waits on nothing") |
| + } |
| + t.Setenv(layoutDefaultEnv, "nestd") |
| + if layoutLegWith(ctx, s, "kass-1", converge) { |
| + t.Fatal("a typo waits on nothing") |
| + } |
| +} |
diff --git a/internal/host/layoutleg.go b/internal/host/layoutleg.go| index 599c10875..a035f412b 100644 |
| --- a/internal/host/layoutleg.go |
| +++ b/internal/host/layoutleg.go |
| @@ -25,6 +25,7 @@ import ( |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| "github.com/carlosframework/platform/internal/status" |
| + "github.com/carlosframework/platform/internal/version" |
| ) |
| // armedLayoutPass is the box's one layout pass. RunAgent stores it once, |
| @@ -53,7 +54,110 @@ func lastLayoutIntent() string { |
| return "" |
| } |
| -func noteLayoutIntent(mode string) { layoutIntentSeen.Store(&mode) } |
| +func noteLayoutIntent(mode string) { |
| + layoutIntentSeen.Store(&mode) |
| + layoutDefaultSeen.Store(&layoutDefaultState{}) |
| +} |
| + |
| +// layoutDefaultEnv is the deployment's answer when no layout intent is |
| +// recorded (isolated-exec design §8.6, "Nested by default"): nested, or |
| +// flat for a deployment that is not ready to isolate its tenants. Read |
| +// from host.env like CARLOS_SYSTEM_CHANNEL, because it is a fact about the |
| +// box's deployment that an operator sets once at bootstrap, and the agent |
| +// reads every such fact the same way. |
| +const layoutDefaultEnv = "CARLOS_LAYOUT_DEFAULT" |
| + |
| +// layoutDefault reads CARLOS_LAYOUT_DEFAULT: "" and "nested" are nested, |
| +// "flat" is flat. Anything else is flat with the raw value returned as |
| +// invalid, so the caller can log it: an operator who typed the variable |
| +// meant to choose, and leaving the box as it was (flat) while saying so is |
| +// the reading that cannot surprise them. |
| +func layoutDefault(getenv func(string) string) (mode string, invalid string) { |
| + v := strings.ToLower(strings.TrimSpace(getenv(layoutDefaultEnv))) |
| + switch v { |
| + case "", release.LayoutModeNested: |
| + return release.LayoutModeNested, "" |
| + case release.LayoutModeFlat: |
| + return release.LayoutModeFlat, "" |
| + } |
| + return release.LayoutModeFlat, v |
| +} |
| + |
| +// layoutDefaultGate is the readiness the default consults before it stands |
| +// in for a nested intent: the same nestedGate the exec backend and the |
| +// layout pass refuse nested work with (instancefs.SelfCheck, the three |
| +// nosymfollow mounts, the control service's probe verdict). Stored when the |
| +// pass is built; nil means no activator is armed, so nothing could wake |
| +// nested and the default does not apply. |
| +var layoutDefaultGate atomic.Pointer[func() error] |
| + |
| +// layoutDefaultState is what status says about the default: Applied when |
| +// the box is nested by default rather than by a recorded intent, Blocked |
| +// with the reason when the deployment's default is nested but this box is |
| +// not ready for it. |
| +type layoutDefaultState struct { |
| + Applied bool |
| + Blocked string |
| +} |
| + |
| +var layoutDefaultSeen atomic.Pointer[layoutDefaultState] |
| + |
| +func lastLayoutDefault() layoutDefaultState { |
| + if d := layoutDefaultSeen.Load(); d != nil { |
| + return *d |
| + } |
| + return layoutDefaultState{} |
| +} |
| + |
| +// layoutBuild is the running build the default's floor check places; |
| +// a var so a test can name a release. |
| +var layoutBuild = func() string { return version.Build } |
| + |
| +// defaultLayoutIntent is the effective intent when none is recorded |
| +// (design §8.6, "Nested by default"). It is nested only on a box that is |
| +// READY: the deployment's default is nested (CARLOS_LAYOUT_DEFAULT), the |
| +// running build is at or above release.LayoutFloor when it can be placed |
| +// (a build that evaluates this rule carries the pass by construction, so |
| +// an unplaceable dev build passes; a mis-tagged release below the floor |
| +// does not), and the nested gate answers: SelfCheck, the mounts and the |
| +// probe verdict. Anything else is "" (flat, as before the default), with |
| +// the reason recorded for status so an operator can see why a box has |
| +// not moved on its own. The default never forces: an awake host gets its |
| +// grace exactly as under a recorded intent. |
| +func defaultLayoutIntent() (mode string, reason string) { |
| + def, invalid := layoutDefault(os.Getenv) |
| + if invalid != "" { |
| + return "", layoutDefaultEnv + "=" + invalid + " is not nested or flat; staying flat" |
| + } |
| + if def != release.LayoutModeNested { |
| + return "", "" |
| + } |
| + if b := layoutBuild(); release.BelowFloor(b, release.LayoutFloor) { |
| + return "", "this box runs " + b + ", below v" + release.LayoutFloor.String() + ", the first release on which nested instances wake" |
| + } |
| + gate := layoutDefaultGate.Load() |
| + if gate == nil { |
| + return "", "no activator is armed on this box, so nothing could wake nested" |
| + } |
| + if err := (*gate)(); err != nil { |
| + return "", "isolation not ready: " + err.Error() |
| + } |
| + return release.LayoutModeNested, "" |
| +} |
| + |
| +// noteLayoutDefault records the default's outcome for status: nested by |
| +// default when mode is nested, otherwise none with the reason it did not |
| +// apply. |
| +func noteLayoutDefault(mode, reason string) { |
| + if mode == release.LayoutModeNested { |
| + layoutIntentSeen.Store(&mode) |
| + layoutDefaultSeen.Store(&layoutDefaultState{Applied: true}) |
| + return |
| + } |
| + none := release.LayoutModeNone |
| + layoutIntentSeen.Store(&none) |
| + layoutDefaultSeen.Store(&layoutDefaultState{Blocked: reason}) |
| +} |
| // layoutStatusReport builds the box's layout report for the status |
| // snapshot (design §8.4, amendment R15): the mode this box last converged |
| @@ -84,6 +188,7 @@ func layoutStatusReport(routes []registry.Route) *status.LayoutReport { |
| if mode == "" { |
| mode = "none" |
| } |
| + def := lastLayoutDefault() |
| rows := map[string]status.RowLayout{} |
| for _, rt := range routes { |
| // Only instance-kind rows: a sidecar (Kind "sidecar") is outside |
| @@ -108,7 +213,7 @@ func layoutStatusReport(routes []registry.Route) *status.LayoutReport { |
| rows[rt.Host] = status.RowLayout{Layout: "external"} |
| } |
| } |
| - return &status.LayoutReport{Mode: mode, Rows: rows} |
| + return &status.LayoutReport{Mode: mode, Rows: rows, Default: def.Applied, DefaultBlocked: def.Blocked} |
| } |
| // LayoutModeUnmanaged is the report Mode of a box with no activator armed: |
| @@ -182,6 +287,18 @@ func socketsGID() int { |
| return n |
| } |
| +// nestedRowsPresent reports whether any activator-managed row on the box is |
| +// already at the nested paths. |
| +func nestedRowsPresent(rows []registry.Route) bool { |
| + dd := provisionTickConfig(os.Getenv).DataDir |
| + for _, rt := range rows { |
| + if activator.ActivatorManaged(rt) && provision.NestedPaths(dd, rt) { |
| + return true |
| + } |
| + } |
| + return false |
| +} |
| + |
| // newLayoutPass builds the pass from what only RunAgent has in hand. |
| // nestedOK must be the same gate the exec backend refuses nested wakes |
| // with, so the pass never makes a nested tenant the box would refuse to |
| @@ -208,6 +325,10 @@ func newLayoutPass(reg *registry.Registry, svc *activator.Service, dd string, ne |
| if svc != nil { |
| p.Fence = svc |
| } |
| + // The default's readiness is this same gate (defaultLayoutIntent). |
| + if nestedOK != nil { |
| + layoutDefaultGate.Store(&nestedOK) |
| + } |
| return p |
| } |
| @@ -248,8 +369,26 @@ func startupLayoutIntent(ctx context.Context, store release.Store, storeErr erro |
| noteLayoutIntent(string(layout.IntentUnreadable)) |
| return layout.IntentUnreadable |
| case !found: |
| - noteLayoutIntent(release.LayoutModeNone) |
| - return "" |
| + // No recorded intent. The default is not asked its gate here: at |
| + // agent start the control service's probe has not run yet (its |
| + // /ready answers "not yet run" until the first async probe lands), |
| + // so the gate would say "not ready" on every start, and recovery |
| + // under "" drops every hold marker (RestoreHolds) and releases a |
| + // completed revert, handing a tenant that kept its move refused a |
| + // fresh grace to run flat. So when the deployment's default is |
| + // nested, startup recovery runs under IntentUnreadable, the |
| + // fail-closed intent: holds and journals are kept, nothing is |
| + // released flat, and the first serial move reads the intent with |
| + // the gate's real answer. The opt-out (CARLOS_LAYOUT_DEFAULT=flat) |
| + // keeps today's behaviour, dropping holds as a recorded flat does. |
| + if def, invalid := layoutDefault(os.Getenv); def == release.LayoutModeNested && invalid == "" { |
| + log.Printf("layout: no intent recorded at start -> holds and journals kept until the first serial move asks the isolation gate (nested by default)") |
| + noteLayoutDefault("", "agent starting: holds and journals kept until the first serial move asks the isolation gate") |
| + return layout.IntentUnreadable |
| + } |
| + mode, reason := defaultLayoutIntent() |
| + noteLayoutDefault(mode, reason) |
| + return layout.Mode(mode) |
| case !release.ValidLayoutMode(in.Mode): |
| log.Printf("layout: intent %s -> unreadable: mode %q is not flat or nested", in.ID, in.Mode) |
| noteLayoutIntent(string(layout.IntentUnreadable)) |
| @@ -306,6 +445,29 @@ type layoutLeg struct { |
| retryPending bool |
| } |
| +// newLayoutLeg builds the leg around one run and, when a pass is already |
| +// armed, kicks it once. The leg otherwise runs only on a serial move or |
| +// its own retry timer, and startup recovery runs fail-closed without |
| +// asking the isolation gate (startupLayoutIntent), so a box that restarts |
| +// and then sees no console bump would keep what startup fenced for as |
| +// long as the fleet stayed quiet. The kick is the first evaluation with |
| +// the gate's real answer, and it depends on nothing but this process. |
| +// fastConvergeLoop builds the leg after startActivator has returned, so |
| +// the pass is armed by then on every box that has an activator; with none |
| +// armed the kick reads nothing (runLayoutLeg). |
| +func newLayoutLeg(ctx context.Context, run func(context.Context) bool) *layoutLeg { |
| + l := &layoutLeg{} |
| + l.run = func(ctx context.Context) { |
| + if run(ctx) { |
| + l.retryLater(ctx, awakeRetry) |
| + } |
| + } |
| + if armedLayoutPass.Load() != nil { |
| + l.kick(ctx) |
| + } |
| + return l |
| +} |
| + |
| // kick asks for a pass and returns at once. |
| func (l *layoutLeg) kick(ctx context.Context) { |
| l.dirty.Store(true) |
| @@ -397,8 +559,31 @@ func layoutLegWith(ctx context.Context, store release.Store, boxLabel string, co |
| noteLayoutIntent(string(layout.IntentUnreadable)) |
| return false |
| case !found: |
| - noteLayoutIntent(release.LayoutModeNone) |
| - return false |
| + // No recorded intent: the deployment's default stands in, on a |
| + // box that is ready for it (design §8.6, "Nested by default"). It |
| + // never forces, so an awake host keeps its grace; a box that is |
| + // not ready converges nothing and status says why. Logged once |
| + // per change of answer, not per serial move. |
| + mode, reason := defaultLayoutIntent() |
| + if prev := lastLayoutDefault(); prev.Blocked != reason || prev.Applied != (mode == release.LayoutModeNested) { |
| + if mode == release.LayoutModeNested { |
| + log.Printf("layout: no intent recorded -> nested by default (%s unset or nested; record `carlos system layout revert` to opt out)", layoutDefaultEnv) |
| + } else if reason != "" { |
| + log.Printf("layout: no intent recorded; nested by default is not in force -> %s", reason) |
| + } |
| + } |
| + noteLayoutDefault(mode, reason) |
| + if mode != release.LayoutModeNested { |
| + // A box the default has not reached yet is retried on the |
| + // leg's own timer (awakeRetry), for the same reason an awake |
| + // host is: the probe that gates it answers in wall time (its |
| + // first run lands seconds after the control service starts), |
| + // and a quiet fleet would otherwise never come back to ask. |
| + // The opt-out and a typo are not waiting on anything. |
| + def, invalid := layoutDefault(os.Getenv) |
| + return def == release.LayoutModeNested && invalid == "" |
| + } |
| + return logLayoutResults(converge(ctx, layout.ModeNested, false)) |
| case !release.ValidLayoutMode(in.Mode): |
| log.Printf("layout: intent %s -> unreadable: mode %q is not flat or nested; not converging this move", in.ID, in.Mode) |
| noteLayoutIntent(string(layout.IntentUnreadable)) |
| @@ -447,11 +632,37 @@ func logLayoutResults(rs []layout.Result) (retry bool) { |
| // rather than minting flat on a box the fleet may want nested. The same |
| // read the layout leg makes, so a mint and a pass agree. |
| func mintLayoutIntent(ctx context.Context, store release.Store) (string, error) { |
| + return mintLayoutIntentWith(ctx, store, nil) |
| +} |
| + |
| +// mintLayoutIntentWith is mintLayoutIntent with the box's own rows in hand |
| +// (nil: no registry to ask). With no intent recorded and the deployment's |
| +// default nested, the default's gate is asked; and when the gate has not |
| +// yet answered (the first reconcile after an agent restart runs before the |
| +// control service's first probe) a box whose registry already holds an |
| +// activator-managed row at the nested paths counts as nested for mints |
| +// all the same: the box has evidently moved, and a flat row minted beside |
| +// nested siblings is the exposure the default exists to end. The gate |
| +// still guards every wake, so a nested row minted this way waits for it. |
| +func mintLayoutIntentWith(ctx context.Context, store release.Store, routes func() ([]registry.Route, error)) (string, error) { |
| in, found, err := release.LoadLayoutIntent(ctx, store, BoxLabel()) |
| switch { |
| case err != nil: |
| return "", err |
| case !found: |
| + mode, _ := defaultLayoutIntent() |
| + if mode == release.LayoutModeNested { |
| + return mode, nil |
| + } |
| + if def, invalid := layoutDefault(os.Getenv); def == release.LayoutModeNested && invalid == "" && routes != nil { |
| + rows, err := routes() |
| + if err != nil { |
| + return "", fmt.Errorf("layout default: list routes: %w", err) |
| + } |
| + if nestedRowsPresent(rows) { |
| + return release.LayoutModeNested, nil |
| + } |
| + } |
| return release.LayoutModeNone, nil |
| case !release.ValidLayoutMode(in.Mode): |
| return "", fmt.Errorf("layout intent %s names mode %q, which this binary does not know", in.ID, in.Mode) |
diff --git a/internal/status/contract_test.go b/internal/status/contract_test.go| index 406d619c9..14c065094 100644 |
| --- a/internal/status/contract_test.go |
| +++ b/internal/status/contract_test.go |
| @@ -86,7 +86,7 @@ var allowedBoxKeys = map[string][]string{ |
| "metrics.route_versions": {wildcardAnyKey}, |
| "metrics.upstreams": {wildcardAnyKey}, |
| "metrics.upstreams.*[]": {"addr", "up", "since", "fails", "last_error"}, |
| - "metrics.layout": {"mode", "rows"}, |
| + "metrics.layout": {"mode", "rows", "default", "default_blocked"}, |
| "metrics.layout.rows": {wildcardAnyKey}, |
| "metrics.layout.rows.*": {"layout", "reason", "note"}, |
| // config_refusals (amadan #213) names hosts and a closed reason word, in |
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go| index 8f8577ff9..161b21fd1 100644 |
| --- a/internal/status/snapshot.go |
| +++ b/internal/status/snapshot.go |
| @@ -180,6 +180,12 @@ type RowLayout struct { |
| type LayoutReport struct { |
| Mode string `json:"mode"` |
| Rows map[string]RowLayout `json:"rows"` |
| + // Default is set when Mode is nested by the deployment's default |
| + // rather than by a recorded intent (design §8.6, "Nested by |
| + // default"); DefaultBlocked is why the default is not in force on a |
| + // box whose Mode is none. Both additive; an old reader ignores them. |
| + Default bool `json:"default,omitempty"` |
| + DefaultBlocked string `json:"default_blocked,omitempty"` |
| } |
| // ConfigRefusal is one host this box is refusing to take because it cannot |
diff --git a/status_instances.go b/status_instances.go| index 93c9404ec..2eb8b11bc 100644 |
| --- a/status_instances.go |
| +++ b/status_instances.go |
| @@ -153,6 +153,10 @@ type statusLayoutBox struct { |
| Stale bool `json:"stale"` |
| Rows []statusLayoutRow `json:"rows"` |
| Mixed bool `json:"mixed"` |
| + // Default: nested by the deployment's default, not a recorded intent; |
| + // DefaultBlocked: why the default is not in force on this box. |
| + Default bool `json:"default"` |
| + DefaultBlocked string `json:"default_blocked"` |
| } |
| func (s *apiSession) statusLayoutList(ctx context.Context) ([]statusLayoutBox, error) { |
| @@ -213,6 +217,12 @@ func printStatusLayout(w io.Writer, boxes []statusLayoutBox) { |
| if b.Stale { |
| name += " (stale)" |
| } |
| + // Nested by default is said on the box, not implied by its rows: |
| + // an operator reading nested rows must be able to tell a box that |
| + // was migrated on purpose from one that moved on its own. |
| + if b.Default { |
| + name += " (nested by default)" |
| + } |
| for _, r := range b.Rows { |
| // A note is an event, not a refusal; it shares the column so |
| // the heading stays the approved one. |
| @@ -249,6 +259,14 @@ func printStatusLayout(w io.Writer, boxes []statusLayoutBox) { |
| fmt.Fprintf(w, "\nWARNING: %s runs nested instances beside flat, foreign or external ones, which share the agent's user and its view of /data. Nested isolation on this box is not in force until none of those remains.\n", b.Box) |
| } |
| } |
| + // A box the default has not reached says why, once per box, so an |
| + // operator waiting for a fleet to move on its own can see what each |
| + // box is waiting for. |
| + for _, b := range boxes { |
| + if b.DefaultBlocked != "" && !b.Stale { |
| + fmt.Fprintf(w, "\n%s is not nested by default: %s\n", b.Box, b.DefaultBlocked) |
| + } |
| + } |
| // Boxes that disagree about their mode are the other loud case: a |
| // box-scoped intent covers the boxes it names, and an instance placed |
| // or relocated onto a box outside it is minted flat there (#126, the |
diff --git a/status_instances_test.go b/status_instances_test.go| index b20239db4..b8d9d09ad 100644 |
| --- a/status_instances_test.go |
| +++ b/status_instances_test.go |
| @@ -179,3 +179,25 @@ func TestPrintStatusLayoutWarnsWhenBoxesDisagreeAboutTheMode(t *testing.T) { |
| t.Errorf("an agreeing fleet warned:\n%s", buf.String()) |
| } |
| } |
| + |
| +// A box nested by default is marked on its name; a box the default has not |
| +// reached says why under the table; a stale box's reason is not repeated. |
| +func TestPrintStatusLayoutSaysNestedByDefaultAndWhyNot(t *testing.T) { |
| + boxes := []statusLayoutBox{ |
| + {Box: "kass-1", Mode: "nested", Default: true, Rows: []statusLayoutRow{{Host: "a.example.com", Layout: "nested"}}}, |
| + {Box: "kass-2", Mode: "none", DefaultBlocked: "isolation not ready: probe: pid1 fail", Rows: []statusLayoutRow{{Host: "b.example.com", Layout: "flat"}}}, |
| + {Box: "kass-3", Mode: "none", Stale: true, DefaultBlocked: "no activator is armed on this box, so nothing could wake nested", Rows: []statusLayoutRow{{Host: "c.example.com", Layout: "flat"}}}, |
| + } |
| + var buf bytes.Buffer |
| + printStatusLayout(&buf, boxes) |
| + out := buf.String() |
| + if !strings.Contains(out, "kass-1 (nested by default) a.example.com nested") { |
| + t.Errorf("nested by default not marked on the box:\n%s", out) |
| + } |
| + if !strings.Contains(out, "kass-2 is not nested by default: isolation not ready: probe: pid1 fail") { |
| + t.Errorf("blocked reason missing:\n%s", out) |
| + } |
| + if strings.Contains(out, "kass-3 is not nested by default") { |
| + t.Errorf("a stale box's reason was repeated:\n%s", out) |
| + } |
| +} |
diff --git a/system.go b/system.go| index 2985a966d..04f98722c 100644 |
| --- a/system.go |
| +++ b/system.go |
| @@ -685,7 +685,9 @@ func systemLayoutDoorError(st int, body []byte) error { |
| // which the line points at. |
| func printSystemLayoutStatus(w io.Writer, out systemLayoutResponse) { |
| if out.Mode == "none" { |
| - fmt.Fprintln(w, "no layout intent recorded; every box keeps the flat layout") |
| + fmt.Fprintln(w, "no layout intent recorded; boxes use the nested layout by default once their isolation is ready (CARLOS_LAYOUT_DEFAULT=flat in a box's host.env opts it out)") |
| + fmt.Fprintln(w, "to choose the layout yourself, record one: carlos system layout migrate [--box <label>] or carlos system layout revert") |
| + fmt.Fprintln(w, "which boxes are nested, and which by default: carlos status --layout") |
| return |
| } |
| scope := "every box" |
| @@ -712,6 +714,11 @@ it, and wait for carlos status --layout to show every instance flat, before |
| rolling the platform back to a release from before the layout. status prints |
| the recorded intent. |
| +With no intent recorded, a box whose isolation is ready uses the nested |
| +layout by default (CARLOS_LAYOUT_DEFAULT in its host.env; flat opts out). |
| +revert is the fleet-wide opt-out; migrate --box is how to move one box at a |
| +time instead. |
| + |
| --box scopes the request to one box, so a fleet moves one box at a time. |
| --force stops an awake instance for its move at once instead of giving it an |
| hour to go to sleep first. Ops account owners and CARLOS_SYSTEM_OPERATORS. |
diff --git a/system_test.go b/system_test.go| index d629eb039..3a0d5f77c 100644 |
| --- a/system_test.go |
| +++ b/system_test.go |
| @@ -253,8 +253,10 @@ func TestPrintSystemAntivirusWarnsOnStaleSignatures(t *testing.T) { |
| func TestPrintSystemLayoutStatus(t *testing.T) { |
| var b strings.Builder |
| printSystemLayoutStatus(&b, systemLayoutResponse{Mode: "none"}) |
| - if !strings.Contains(b.String(), "no layout intent recorded") { |
| - t.Errorf("none: %q", b.String()) |
| + for _, want := range []string{"no layout intent recorded; boxes use the nested layout by default once their isolation is ready", "CARLOS_LAYOUT_DEFAULT=flat", "carlos system layout revert", "carlos status --layout"} { |
| + if !strings.Contains(b.String(), want) { |
| + t.Errorf("none lacks %q: %q", want, b.String()) |
| + } |
| } |
| b.Reset() |
| printSystemLayoutStatus(&b, systemLayoutResponse{Mode: "nested", Box: "kass-1", RequestedAt: "2026-10-06T10:00:00Z", Force: true}) |