amadan

carlos / platform Public

Sidecar isolation Task 9 (3/5): the agent arms the policy, keeps the spawn record and reports the inventory (D12, D14, D15, D16)

CARLOS_FLAT_EXEC and CARLOS_FLAT_EXEC_ALLOW are read once at agent start
(unset is report, unknown is refuse, exact hosts only); the policy goes on
the exec backend. Each status snapshot builds the inventory of what refuse
would stop from its own registry read through the armed pass's Classify
(R4-3), never from the layout pass's run; with no pass armed it is unknown,
which never counts as empty or closed. Status also carries the effective
intent word, the per-host spawn record, D15's closure check and the
workers readiness word (nested only when the readiness, the intent, the
gate and claim mode all allow it). A refused flat row on a nested box kicks
the layout leg (D16); a foreign row is not nudged.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
claude e0ff1412b32f9b26cdd4cbd776f44a862881b8c4 parent f7efe00
5 files changed, +779 −0
  • internal/host/agent.go +19 −0
  • internal/host/flatexec.go +345 −0
  • internal/host/flatexec_copy.go +9 −0
  • internal/host/flatexec_test.go +396 −0
  • internal/host/layoutleg.go +10 −0
diff --git a/internal/host/agent.go b/internal/host/agent.go
index ebcfc3141..1c3d5057d 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -248,6 +248,13 @@ func runAgent(args []string, buildClaims ClaimFactory) {
} else {
log.Printf("agent: workers readiness %s: %s", w.Word(), w.Reason())
}
+ // The flat-exec policy is read here too, once, for the life of this
+ // agent (sidecar isolation D12, D15): refuse or report, and the exact
+ // hosts allowed to run in the host namespace. It goes on the exec
+ // backend below; a refusal of a managed row on a nested box kicks the
+ // layout leg (D16) once fastConvergeLoop has built it.
+ flatExec := armFlatExec(os.Getenv, nudgeLayoutLeg)
+ log.Printf("agent: flat exec %s (%d hosts allowlisted)", armedFlatExec.Load().mode, len(flatExec.Allow))
regPath := peekRegistryPath(args, dd)
// A second handle on the same registry file the edge opens — the registry
// is designed for concurrent handles (fresh-connection reads), and this
@@ -443,6 +450,9 @@ func runAgent(args []string, buildClaims ClaimFactory) {
UpdatesReader: updatesReader(reg, release.OptDir()),
// Asked inside Acquire, before the lease (amadan #213).
ConfigGate: configGate(),
+ // The flat-exec policy (sidecar isolation D13), asked at both
+ // spawn seams before any cgroup is touched.
+ FlatExec: flatExec,
}
if claims.Writes.Enabled {
eb.Timers = schedule.ClaimTimers{TimerStore: reg, Writes: claims.Writes, Context: agentCtx}
@@ -3104,6 +3114,11 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service, sd *schedu
layoutBox := metricsBoxLabel(os.Getenv, readMachineID())
layoutPasses := newLayoutLeg(ctx, func(ctx context.Context) bool { return runLayoutLeg(ctx, store, layoutBox) })
d.layoutLeg = layoutPasses.kick
+ // The flat-exec policy's D16 nudge: a refused managed row on a nested
+ // box kicks this same leg, so the pass runs within seconds, finds the
+ // host asleep and moves it.
+ kickLayout := func() { layoutPasses.kick(ctx) }
+ layoutNudge.Store(&kickLayout)
// The platform self-update leg (spec §4): the sandboxed half, which only
// ever fetches into /data and writes a request. It lives here rather than
// in a loop of its own because the spread deadline needs a seconds
@@ -4093,6 +4108,10 @@ func statusLoop(reg *registry.Registry, svc *activator.Service, buildStates func
// pass still says something rather than nothing.
Layout: layoutStatusReport(routes),
}
+ // The flat-exec policy, its spawn record and the inventory of what
+ // refuse would stop (sidecar isolation D14), from this same
+ // registry read (R4-3). Private Metrics only: hosts.
+ p.FlatExec = flatExecStatusReport(routes)
// The hosts this box is refusing because it cannot provide their
// config (amadan #213): private Metrics only, nothing public.
claimReport(&p)
diff --git a/internal/host/flatexec.go b/internal/host/flatexec.go
new file mode 100644
index 000000000..0fb7db9fe
--- /dev/null
+++ b/internal/host/flatexec.go
@@ -0,0 +1,345 @@
+package host
+
+import (
+ "log"
+ "os"
+ "sort"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+// The flat-exec policy on the box (sidecar isolation spec rev 4, D12 to
+// D16): how the agent reads CARLOS_FLAT_EXEC and CARLOS_FLAT_EXEC_ALLOW
+// once at start, keeps the spawn record the policy reports into, builds
+// the inventory of what refuse would stop for each status snapshot, and
+// nudges the layout pass when a managed row is refused on a nested box.
+// Both settings live in root-owned host.env; no file under /data decides
+// any of this (D15).
+
+// The env names and the policy words (D12).
+const (
+ flatExecEnv = "CARLOS_FLAT_EXEC"
+ flatExecAllowEnv = "CARLOS_FLAT_EXEC_ALLOW"
+ flatExecRefuse = "refuse"
+ flatExecReport = "report"
+)
+
+// The closed words for the effective layout intent with its source (D14):
+// nested; flat because the fleet or the deployment chose it (a recorded
+// flat intent, or CARLOS_LAYOUT_DEFAULT=flat); flat because the box fell
+// back to it (the gate failing with no intent recorded, an invalid default,
+// the agent still starting); unknown because the last intent read failed
+// or no evaluation has run yet.
+const (
+ intentNested = "nested"
+ intentFlatChoice = "flat_choice"
+ intentFlatFallback = "flat_fallback"
+ intentUnknown = "unknown"
+)
+
+// workersBlocked is the readiness word status decides itself (D14): the
+// start-time readiness holds but the box's intent, gate or mode does not
+// let a worker go nested.
+const workersBlocked = "blocked"
+
+// flatExecFromEnv reads the setting the way CARLOS_LAYOUT_DEFAULT is read
+// (trimmed, lower-cased): refuse or report; unset is report (rev 4, D15:
+// no automatic change); an unknown value is refuse, because an operator
+// who typed the variable meant to choose and the reading that cannot
+// expose a tenant is the one to take. allow is CARLOS_FLAT_EXEC_ALLOW's
+// exact hosts, each a valid host shape (provision.ValidHostShape), sorted
+// and deduplicated; dropped is every entry that was not, kept so status
+// and the log can name the typo. No wildcards or suffixes: one entry
+// reopening *.sidecar.internal would undo the whole change, and a star
+// is not a valid host shape.
+func flatExecFromEnv(getenv func(string) string) (mode string, allow []string, dropped []string) {
+ switch v := strings.ToLower(strings.TrimSpace(getenv(flatExecEnv))); v {
+ case "", flatExecReport:
+ mode = flatExecReport
+ case flatExecRefuse:
+ mode = flatExecRefuse
+ default:
+ mode = flatExecRefuse
+ log.Printf("agent: %s=%q is not refuse or report; refusing flat spawns", flatExecEnv, v)
+ }
+ seen := map[string]bool{}
+ for _, raw := range strings.Split(getenv(flatExecAllowEnv), ",") {
+ h := strings.ToLower(strings.TrimSpace(raw))
+ if h == "" {
+ continue
+ }
+ if !provision.ValidHostShape(h) {
+ dropped = append(dropped, h)
+ continue
+ }
+ if !seen[h] {
+ seen[h] = true
+ allow = append(allow, h)
+ }
+ }
+ sort.Strings(allow)
+ if len(dropped) > 0 {
+ log.Printf("agent: %s: %d entries are not exact hosts and were dropped: %s", flatExecAllowEnv, len(dropped), strings.Join(dropped, ", "))
+ }
+ return mode, allow, dropped
+}
+
+// flatExecRecord is one host's spawn record (D14), the shape ConfigRefusal
+// has: the last reason word, the first and last time, and a count.
+type flatExecRecord struct {
+ reason string
+ since time.Time
+ last time.Time
+ count int
+}
+
+// flatExecState is the policy as armed for the life of the agent, with the
+// spawn record the policy's Record fills in. One per process
+// (armedFlatExec); a test arms its own and resets it.
+type flatExecState struct {
+ mode string
+ allow map[string]bool
+ allowed []string // sorted, for status
+ dropped []string
+
+ // now and nudge are seams: the clock for the record, and the layout
+ // leg's kick for D16. Both set by armFlatExec; a test replaces them.
+ now func() time.Time
+ nudge func()
+
+ mu sync.Mutex
+ records map[string]*flatExecRecord
+}
+
+var armedFlatExec atomic.Pointer[flatExecState]
+
+// layoutNudge is the layout leg's kick, published by fastConvergeLoop once
+// the leg exists (the policy is armed before it, at agent start). Nil
+// until then: a refusal in the window before the leg is built is not
+// nudged, and the leg's own first kick covers it.
+var layoutNudge atomic.Pointer[func()]
+
+// nudgeLayoutLeg is the policy's D16 nudge: the published kick, if any.
+func nudgeLayoutLeg() {
+ if f := layoutNudge.Load(); f != nil {
+ (*f)()
+ }
+}
+
+// flatExecRecordsCap bounds the spawn record. Hosts come from the registry,
+// which bounds them already; the cap is for a box whose registry churns
+// hosts faster than records age, so the record cannot grow without bound
+// over months of uptime. The oldest record goes first.
+const flatExecRecordsCap = 500
+
+// armFlatExec reads the setting and arms the policy for the life of this
+// agent. nudge is the layout leg's kick (D16), asked when a managed row is
+// refused on a box whose effective intent is nested; nil is no nudge.
+// The returned policy goes on the exec backend; nothing else holds it.
+func armFlatExec(getenv func(string) string, nudge func()) *activator.FlatExecPolicy {
+ mode, allow, dropped := flatExecFromEnv(getenv)
+ s := &flatExecState{mode: mode, allow: map[string]bool{}, allowed: allow, dropped: dropped, now: time.Now, nudge: nudge, records: map[string]*flatExecRecord{}}
+ for _, h := range allow {
+ s.allow[h] = true
+ }
+ armedFlatExec.Store(s)
+ return s.policy()
+}
+
+// policy is the activator's view of the state: refuse or report, the
+// allowlist, and the record hook.
+func (s *flatExecState) policy() *activator.FlatExecPolicy {
+ return &activator.FlatExecPolicy{Refuse: s.mode == flatExecRefuse, Allow: s.allow, Record: s.record}
+}
+
+// record is the policy's Record: it files the host's refusal or would-be
+// refusal, and under a refusal of a flat row on a box whose effective
+// intent is nested it kicks the layout pass (D16), which finds the host
+// asleep and moves it, so the next wake is nested. A foreign row is not
+// nudged: the pass does not move foreign rows, and its repair is the
+// box-local carlos add.
+func (s *flatExecState) record(host, reason string) {
+ now := s.now()
+ s.mu.Lock()
+ r := s.records[host]
+ if r == nil {
+ if len(s.records) >= flatExecRecordsCap {
+ s.evictOldestLocked()
+ }
+ r = &flatExecRecord{since: now}
+ s.records[host] = r
+ }
+ if r.reason != reason {
+ // A new stretch: the reason changed (report became refuse at a
+ // restart, or a row was repaired from foreign to flat).
+ r.reason, r.since, r.count = reason, now, 0
+ }
+ r.last = now
+ r.count++
+ s.mu.Unlock()
+ if reason == activator.FlatExecFlatRefused && s.nudge != nil && effectiveLayoutIntent(os.Getenv) == intentNested {
+ s.nudge()
+ }
+}
+
+func (s *flatExecState) evictOldestLocked() {
+ var oldest string
+ var at time.Time
+ for h, r := range s.records {
+ if oldest == "" || r.last.Before(at) {
+ oldest, at = h, r.last
+ }
+ }
+ delete(s.records, oldest)
+}
+
+// refusals is the spawn record for status, sorted by host.
+func (s *flatExecState) refusals() []status.FlatExecRefusal {
+ s.mu.Lock()
+ defer s.mu.Unlock()
+ out := make([]status.FlatExecRefusal, 0, len(s.records))
+ for h, r := range s.records {
+ out = append(out, status.FlatExecRefusal{Host: h, Reason: r.reason, Since: r.since, Last: r.last, Count: r.count})
+ }
+ sort.Slice(out, func(i, j int) bool { return out[i].Host < out[j].Host })
+ return out
+}
+
+// InventoryRow is one row of what refuse would stop (D14).
+type InventoryRow = status.FlatExecRow
+
+// flatExecInventory is D14's inventory, from the registry rather than from
+// history: every row the agent could spawn through a seam (an
+// activator-managed instance row or a worker row, foreign ones included,
+// awake or asleep) that classify calls flat or foreign and that allow does
+// not name, by host and layout word, sorted by host. A dormant row that
+// has never woken is in it like any other.
+func flatExecInventory(routes []registry.Route, classify func(registry.Route) activator.Layout, allow map[string]bool) []InventoryRow {
+ var rows []InventoryRow
+ for _, rt := range routes {
+ if !activator.ActivatorManaged(rt) && !activator.WorkerManaged(rt) {
+ continue
+ }
+ if allow[rt.Host] {
+ continue
+ }
+ l := classify(rt)
+ if l != activator.LayoutFlat && l != activator.LayoutForeign {
+ continue
+ }
+ rows = append(rows, InventoryRow{Host: rt.Host, Layout: string(l), Worker: rt.IsSidecar()})
+ }
+ sort.Slice(rows, func(i, j int) bool { return rows[i].Host < rows[j].Host })
+ return rows
+}
+
+// flatExecStatusReport builds the box's flat-exec report for the status
+// snapshot from the routes the caller already listed (R4-3: the snapshot's
+// own registry read, never the layout pass's run, which returns early on a
+// flat default, an unreadable intent or a failed gate). nil when no policy
+// is armed, which is no agent at all. The inventory needs the armed pass's
+// Classify, the same classification a wake sees; with no pass armed it is
+// unknown, and an unknown inventory never counts as empty or closed.
+func flatExecStatusReport(routes []registry.Route) *status.FlatExecReport {
+ s := armedFlatExec.Load()
+ if s == nil {
+ return nil
+ }
+ r := &status.FlatExecReport{
+ Mode: s.mode,
+ Allow: s.allowed,
+ Dropped: s.dropped,
+ Intent: effectiveLayoutIntent(os.Getenv),
+ Refusals: s.refusals(),
+ }
+ pass := armedLayoutPass.Load()
+ if pass == nil {
+ r.InventoryUnknown = "no layout pass is armed on this box, so its rows cannot be classified"
+ return r
+ }
+ classify := func(rt registry.Route) activator.Layout {
+ l, _ := pass.Classify(rt, routes)
+ return l
+ }
+ r.Inventory = flatExecInventory(routes, classify, s.allow)
+ workersListed := false
+ for _, row := range r.Inventory {
+ if row.Worker {
+ workersListed = true
+ }
+ }
+ for _, rt := range routes {
+ if activator.WorkerManaged(rt) && classify(rt) == activator.LayoutFlat {
+ r.FlatWorkers++
+ }
+ }
+ // D15's closure check: refuse in force, nothing allowlisted, the
+ // inventory known (it is, here), and no worker row flat or foreign.
+ r.Closed = s.mode == flatExecRefuse && len(s.allow) == 0 && !workersListed
+ return r
+}
+
+// effectiveLayoutIntent is the box's layout intent with its source, as
+// the closed words of D14, read from what the layout leg last published
+// (layoutIntentSeen, layoutDefaultSeen) and the deployment's default.
+func effectiveLayoutIntent(getenv func(string) string) string {
+ switch lastLayoutIntent() {
+ case release.LayoutModeNested:
+ return intentNested
+ case release.LayoutModeFlat:
+ return intentFlatChoice
+ case release.LayoutModeNone:
+ default:
+ // IntentUnreadable, or a mode this binary does not know.
+ return intentUnknown
+ }
+ // No intent recorded (or none read yet). The deployment's default
+ // stands in: flat by choice when it says flat; nested when the leg
+ // applied it (which publishes nested above); otherwise the box is flat
+ // because the default could not take, or has not been asked yet.
+ def, invalid := layoutDefault(getenv)
+ if invalid != "" {
+ return intentFlatFallback
+ }
+ if def == release.LayoutModeFlat {
+ return intentFlatChoice
+ }
+ if lastLayoutDefault().Blocked != "" {
+ return intentFlatFallback
+ }
+ return intentUnknown
+}
+
+// workersStatusWord is Layout.Workers and its reason (D14): the cached
+// start-time readiness word unless it holds, and then nested only when the
+// effective intent is nested, the nested gate answers and claim mode is
+// off; otherwise blocked, with the first of those that does not hold as
+// the private reason.
+func workersStatusWord(getenv func(string) string) (word, reason string) {
+ w := agentWorkers()
+ if !w.Ready() {
+ return w.Word(), w.Reason()
+ }
+ if strings.TrimSpace(getenv("CARLOS_HOST_CLAIM")) != "" {
+ return workersBlocked, "claim mode is on; the nested worker runs on the legacy path only"
+ }
+ if intent := effectiveLayoutIntent(getenv); intent != intentNested {
+ return workersBlocked, "the effective layout intent is " + intent + ", not nested"
+ }
+ gate := layoutDefaultGate.Load()
+ if gate == nil {
+ return workersBlocked, "no activator is armed on this box, so nothing could wake nested"
+ }
+ if err := (*gate)(); err != nil {
+ return workersBlocked, "isolation not ready: " + err.Error()
+ }
+ return w.Word(), ""
+}
diff --git a/internal/host/flatexec_copy.go b/internal/host/flatexec_copy.go
new file mode 100644
index 000000000..04de6e379
--- /dev/null
+++ b/internal/host/flatexec_copy.go
@@ -0,0 +1,9 @@
+package host
+
+// Copy for the flat-exec policy's operator surfaces (sidecar isolation spec
+// D12, D14), collected and reviewed in Task 12. A placeholder until then.
+
+// flatExecAllowlistedWarning is what `carlos status --layout` says beside
+// every allowlisted host: it runs as the agent in the host namespace and
+// is not isolated from its neighbours.
+const flatExecAllowlistedWarning = "unisolated: allowlisted in CARLOS_FLAT_EXEC_ALLOW, this host runs as the agent in the host namespace and can read every tenant on the box" // copy: placeholder, Task 12
diff --git a/internal/host/flatexec_test.go b/internal/host/flatexec_test.go
new file mode 100644
index 000000000..b225906bd
--- /dev/null
+++ b/internal/host/flatexec_test.go
@@ -0,0 +1,396 @@
+package host
+
+import (
+ "errors"
+ "path/filepath"
+ "reflect"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/layout"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// resetFlatExec puts the armed policy, the leg's nudge and the workers
+// readiness back the way a fresh process has them.
+func resetFlatExec(t *testing.T) {
+ t.Helper()
+ prevWorkers := startWorkers.Load()
+ clear := func() {
+ armedFlatExec.Store(nil)
+ layoutNudge.Store(nil)
+ }
+ clear()
+ t.Cleanup(func() {
+ clear()
+ startWorkers.Store(prevWorkers)
+ })
+}
+
+// TestFlatExecEnvTable pins D12's reading: refuse and report by name, unset
+// is report (rev 4, D15: nothing changes until an operator sets it), an
+// unknown value is refuse, the allowlist is exact hosts only (an invalid
+// entry or a wildcard is dropped and reported), sorted and deduplicated.
+func TestFlatExecEnvTable(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ env map[string]string
+ mode string
+ allow []string
+ dropped []string
+ }{
+ {name: "unset is report", env: nil, mode: "report"},
+ {name: "report", env: map[string]string{"CARLOS_FLAT_EXEC": "report"}, mode: "report"},
+ {name: "refuse", env: map[string]string{"CARLOS_FLAT_EXEC": "refuse"}, mode: "refuse"},
+ {name: "trimmed and lower-cased", env: map[string]string{"CARLOS_FLAT_EXEC": " Refuse\n"}, mode: "refuse"},
+ {name: "unknown is refuse", env: map[string]string{"CARLOS_FLAT_EXEC": "allow"}, mode: "refuse"},
+ {name: "exact hosts", env: map[string]string{"CARLOS_FLAT_EXEC_ALLOW": "tools.example.com, shop.example.com,shop.example.com"}, mode: "report", allow: []string{"shop.example.com", "tools.example.com"}},
+ {name: "wildcards rejected", env: map[string]string{"CARLOS_FLAT_EXEC_ALLOW": "*.sidecar.internal,shop.example.com,.example.com"}, mode: "report", allow: []string{"shop.example.com"}, dropped: []string{"*.sidecar.internal", ".example.com"}},
+ {name: "invalid entry dropped and reported", env: map[string]string{"CARLOS_FLAT_EXEC": "refuse", "CARLOS_FLAT_EXEC_ALLOW": "shop.example.com,not a host,,"}, mode: "refuse", allow: []string{"shop.example.com"}, dropped: []string{"not a host"}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ mode, allow, dropped := flatExecFromEnv(envOf(tc.env))
+ if mode != tc.mode || !reflect.DeepEqual(allow, tc.allow) || !reflect.DeepEqual(dropped, tc.dropped) {
+ t.Fatalf("flatExecFromEnv = %q %v %v, want %q %v %v", mode, allow, dropped, tc.mode, tc.allow, tc.dropped)
+ }
+ })
+ }
+ // Armed, the policy carries exactly the reading: refuse, the allowlist
+ // as a set, and the record hook.
+ resetFlatExec(t)
+ p := armFlatExec(envOf(map[string]string{"CARLOS_FLAT_EXEC": "refuse", "CARLOS_FLAT_EXEC_ALLOW": "shop.example.com"}), nil)
+ if !p.Refuse || !p.Allow["shop.example.com"] || len(p.Allow) != 1 || p.Record == nil {
+ t.Fatalf("armed policy = %+v", p)
+ }
+ if err := p.Admit(registry.Route{Host: "shop.example.com"}, activator.LayoutFlat); err != nil {
+ t.Fatalf("allowlisted host: %v", err)
+ }
+ if err := p.Admit(registry.Route{Host: "other.example.com"}, activator.LayoutFlat); !errors.Is(err, activator.ErrFlatExecRefused) {
+ t.Fatalf("other host: %v", err)
+ }
+ s := armedFlatExec.Load()
+ if got := s.refusals(); len(got) != 1 || got[0].Host != "other.example.com" || got[0].Reason != activator.FlatExecFlatRefused || got[0].Count != 1 {
+ t.Fatalf("spawn record = %+v", got)
+ }
+}
+
+// flatExecFixture arms a pass over a registry with one row of every shape
+// the inventory must judge: a dormant flat instance that has never woken, a
+// foreign instance (carlos add --db), a flat worker, a foreign worker, a
+// nested instance, a nested worker, an external socket row and a
+// unit-backed worker. It returns the routes as the status loop lists them.
+type flatExecFixtureRows struct {
+ dormant, foreign, flatWorker, foreignWorker, nested, nestedWorker string
+ routes []registry.Route
+}
+
+func flatExecFixture(t *testing.T) flatExecFixtureRows {
+ t.Helper()
+ resetLayoutGlobals(t)
+ dd, sd := t.TempDir(), t.TempDir()
+ reg := restartTestRegistry(t)
+ armedLayoutPass.Store(&layout.Pass{Registry: reg, DataDir: dd, SocketDir: sd})
+ f := flatExecFixtureRows{
+ dormant: "dormant.example.com",
+ foreign: "added.example.com",
+ flatWorker: "flat.shop.abc123" + provision.SidecarSuffix,
+ foreignWorker: "edited.shop.abc123" + provision.SidecarSuffix,
+ nested: "nested.example.com",
+ nestedWorker: "nested.shop.abc123" + provision.SidecarSuffix,
+ }
+ f.routes = []registry.Route{
+ // Never woken: no status beyond ready, no process ever started.
+ {Host: f.dormant, Kind: "instance", Hibernate: true, Status: "ready", Socket: filepath.Join(sd, f.dormant+".sock"), DBPath: provision.DefaultDBPath(dd, f.dormant)},
+ {Host: f.foreign, Kind: "instance", Hibernate: true, Status: "ready", Socket: filepath.Join(sd, f.foreign+".sock"), DBPath: filepath.Join(dd, "handmade.db")},
+ {Host: f.flatWorker, Kind: "sidecar", Hibernate: true, Status: "ready", DBPath: provision.DefaultDBPath(dd, f.flatWorker)},
+ {Host: f.foreignWorker, Kind: "sidecar", Status: "ready", DBPath: filepath.Join(dd, "tenant-wrote-this.db")},
+ {Host: f.nested, Kind: "instance", Hibernate: true, Status: "ready", Socket: filepath.Join(sd, "instances", provision.InstanceID(f.nested), "app.sock"), DBPath: provision.NestedDBPath(dd, f.nested)},
+ {Host: f.nestedWorker, Kind: "sidecar", Status: "ready", DBPath: provision.NestedDBPath(dd, f.nestedWorker)},
+ {Host: "external.example.com", Kind: "instance", Socket: filepath.Join(sd, "external.example.com.sock"), DBPath: provision.DefaultDBPath(dd, "external.example.com")},
+ {Host: "unit.shop.abc123" + provision.SidecarSuffix, Kind: "sidecar", Backing: registry.BackingUnit, DBPath: provision.DefaultDBPath(dd, "unit.shop.abc123"+provision.SidecarSuffix)},
+ }
+ for _, rt := range f.routes {
+ if err := reg.Upsert(rt); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return f
+}
+
+func inventoryHosts(rows []InventoryRow) map[string]string {
+ out := map[string]string{}
+ for _, r := range rows {
+ out[r.Host] = r.Layout
+ }
+ return out
+}
+
+// A dormant flat row that has never woken is in the inventory on the first
+// snapshot (D14: from the registry, not from history), beside the flat and
+// foreign worker rows and the foreign instance row; nested, external and
+// unit-backed rows are not.
+func TestInventoryIncludesDormantRow(t *testing.T) {
+ resetFlatExec(t)
+ f := flatExecFixture(t)
+ armFlatExec(envOf(nil), nil)
+ r := flatExecStatusReport(f.routes)
+ if r == nil || r.InventoryUnknown != "" {
+ t.Fatalf("report = %+v", r)
+ }
+ want := map[string]string{f.dormant: "flat", f.foreign: "foreign", f.flatWorker: "flat", f.foreignWorker: "foreign"}
+ if got := inventoryHosts(r.Inventory); !reflect.DeepEqual(got, want) {
+ t.Fatalf("inventory = %v, want %v", got, want)
+ }
+ for _, row := range r.Inventory {
+ if row.Worker != strings.HasSuffix(row.Host, provision.SidecarSuffix) {
+ t.Errorf("%s worker = %t", row.Host, row.Worker)
+ }
+ }
+ if r.FlatWorkers != 1 {
+ t.Errorf("flat workers = %d, want 1", r.FlatWorkers)
+ }
+ if r.Mode != "report" || r.Closed {
+ t.Errorf("mode %q closed %t under an unset setting", r.Mode, r.Closed)
+ }
+}
+
+// Review Focus 2: a box with a foreign row (carlos add --db) before refuse.
+// Status lists it in the inventory with its layout word, and once a spawn
+// of it has been recorded, the spawn record names it with the reported
+// reason, so the operator sees it before setting refuse.
+func TestStatusListsForeignRowInInventory(t *testing.T) {
+ resetFlatExec(t)
+ f := flatExecFixture(t)
+ p := armFlatExec(envOf(map[string]string{"CARLOS_FLAT_EXEC": "report"}), nil)
+ foreign := f.routes[1]
+ if err := p.Admit(foreign, activator.LayoutForeign); err != nil {
+ t.Fatalf("report admitted nothing: %v", err)
+ }
+ r := flatExecStatusReport(f.routes)
+ if got := inventoryHosts(r.Inventory)[f.foreign]; got != "foreign" {
+ t.Fatalf("foreign row in inventory = %q, want foreign (inventory %v)", got, r.Inventory)
+ }
+ if len(r.Refusals) != 1 || r.Refusals[0].Host != f.foreign || r.Refusals[0].Reason != activator.FlatExecForeignReported || r.Refusals[0].Count != 1 {
+ t.Fatalf("spawn record = %+v", r.Refusals)
+ }
+ // The allowlist takes a row out of the inventory, and the allowlist is
+ // reported beside it.
+ resetFlatExec(t)
+ armFlatExec(envOf(map[string]string{"CARLOS_FLAT_EXEC_ALLOW": f.foreign}), nil)
+ r = flatExecStatusReport(f.routes)
+ if _, listed := inventoryHosts(r.Inventory)[f.foreign]; listed || !reflect.DeepEqual(r.Allow, []string{f.foreign}) {
+ t.Fatalf("allowlisted foreign row: inventory %v allow %v", r.Inventory, r.Allow)
+ }
+}
+
+// D15's closure check is true only with refuse in force, an empty
+// allowlist, a known inventory and no worker row flat or foreign. An
+// allowlisted worker does not close it; a flat instance row alone does.
+func TestClosureCheck(t *testing.T) {
+ resetFlatExec(t)
+ f := flatExecFixture(t)
+ nestedOnly := []registry.Route{f.routes[4], f.routes[5], f.routes[0]} // nested instance, nested worker, a flat instance row
+ for _, tc := range []struct {
+ name string
+ env map[string]string
+ routes []registry.Route
+ closed bool
+ }{
+ {name: "report with no flat worker", env: map[string]string{"CARLOS_FLAT_EXEC": "report"}, routes: nestedOnly},
+ {name: "refuse with a flat worker", env: map[string]string{"CARLOS_FLAT_EXEC": "refuse"}, routes: f.routes},
+ {name: "refuse with only a foreign worker", env: map[string]string{"CARLOS_FLAT_EXEC": "refuse"}, routes: append(nestedOnly, f.routes[3])},
+ {name: "refuse with the flat worker allowlisted", env: map[string]string{"CARLOS_FLAT_EXEC": "refuse", "CARLOS_FLAT_EXEC_ALLOW": f.flatWorker}, routes: append(nestedOnly, f.routes[2])},
+ {name: "refuse, empty allowlist, no flat or foreign worker", env: map[string]string{"CARLOS_FLAT_EXEC": "refuse"}, routes: nestedOnly, closed: true},
+ {name: "refuse with no rows at all", env: map[string]string{"CARLOS_FLAT_EXEC": "refuse"}, routes: nil, closed: true},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ armedFlatExec.Store(nil)
+ armFlatExec(envOf(tc.env), nil)
+ r := flatExecStatusReport(tc.routes)
+ if r.Closed != tc.closed {
+ t.Fatalf("closed = %t, want %t (report %+v)", r.Closed, tc.closed, r)
+ }
+ })
+ }
+}
+
+// R4-3: the inventory is computed from the snapshot's registry read under
+// every intent state the layout pass returns early on: a flat default, an
+// unreadable intent, a failed gate. Each is reported with its intent word.
+func TestInventoryComputedUnderFlatDefault(t *testing.T) {
+ resetFlatExec(t)
+ f := flatExecFixture(t)
+ t.Setenv(layoutDefaultEnv, "flat")
+ armFlatExec(envOf(nil), nil)
+ r := flatExecStatusReport(f.routes)
+ if r.Intent != intentFlatChoice || len(r.Inventory) != 4 || r.InventoryUnknown != "" {
+ t.Fatalf("under a flat default: %+v", r)
+ }
+}
+
+func TestInventoryComputedUnderUnreadableIntent(t *testing.T) {
+ resetFlatExec(t)
+ f := flatExecFixture(t)
+ noteLayoutIntent(string(layout.IntentUnreadable))
+ armFlatExec(envOf(nil), nil)
+ r := flatExecStatusReport(f.routes)
+ if r.Intent != intentUnknown || len(r.Inventory) != 4 || r.InventoryUnknown != "" {
+ t.Fatalf("under an unreadable intent: %+v", r)
+ }
+}
+
+func TestInventoryComputedUnderFailedGate(t *testing.T) {
+ resetFlatExec(t)
+ f := flatExecFixture(t)
+ noteLayoutDefault("", "isolation not ready: the probe has not run")
+ armFlatExec(envOf(nil), nil)
+ r := flatExecStatusReport(f.routes)
+ if r.Intent != intentFlatFallback || len(r.Inventory) != 4 || r.InventoryUnknown != "" {
+ t.Fatalf("under a failed gate: %+v", r)
+ }
+ // And the two intents that are a choice or nested.
+ noteLayoutIntent(release.LayoutModeNested)
+ if r := flatExecStatusReport(f.routes); r.Intent != intentNested || len(r.Inventory) != 4 {
+ t.Fatalf("under a nested intent: %+v", r)
+ }
+ noteLayoutIntent(release.LayoutModeFlat)
+ if r := flatExecStatusReport(f.routes); r.Intent != intentFlatChoice {
+ t.Fatalf("under a recorded flat intent: %+v", r)
+ }
+}
+
+// An inventory that could not be computed (no pass armed, so no
+// classification) is unknown, and unknown never counts as empty or closed,
+// however the policy is set.
+func TestInventoryUnknownIsNotClosed(t *testing.T) {
+ resetFlatExec(t)
+ resetLayoutGlobals(t)
+ armFlatExec(envOf(map[string]string{"CARLOS_FLAT_EXEC": "refuse"}), nil)
+ r := flatExecStatusReport(nil)
+ if r == nil || r.InventoryUnknown == "" || r.Closed || len(r.Inventory) != 0 || r.Mode != "refuse" {
+ t.Fatalf("with no pass armed: %+v", r)
+ }
+ // No policy armed at all (no agent): nothing to report.
+ armedFlatExec.Store(nil)
+ if r := flatExecStatusReport(nil); r != nil {
+ t.Fatalf("unarmed policy reported %+v", r)
+ }
+}
+
+// D16: a refusal of a flat (managed) row on a box whose effective intent is
+// nested kicks the layout leg; a foreign row's refusal does not, nor does a
+// would-be refusal under report, nor any refusal on a box whose intent is
+// not nested.
+func TestRefusedManagedRowNudgesPass(t *testing.T) {
+ resetFlatExec(t)
+ resetLayoutGlobals(t)
+ kicks := 0
+ armFlatExec(envOf(map[string]string{"CARLOS_FLAT_EXEC": "refuse"}), func() { kicks++ })
+ s := armedFlatExec.Load()
+ noteLayoutIntent(release.LayoutModeNested)
+ s.record("shop.example.com", activator.FlatExecFlatRefused)
+ if kicks != 1 {
+ t.Fatalf("flat refusal under nested: kicks = %d, want 1", kicks)
+ }
+ s.record("added.example.com", activator.FlatExecForeignRefused)
+ s.record("shop.example.com", activator.FlatExecFlatReported)
+ if kicks != 1 {
+ t.Fatalf("foreign refusal or a report nudged: kicks = %d", kicks)
+ }
+ noteLayoutIntent(release.LayoutModeFlat)
+ s.record("shop.example.com", activator.FlatExecFlatRefused)
+ if kicks != 1 {
+ t.Fatalf("flat refusal under a flat intent nudged: kicks = %d", kicks)
+ }
+ // The published kick is what the agent's nudge reaches; nil before the
+ // leg exists is a quiet no-op.
+ nudgeLayoutLeg()
+ published := 0
+ f := func() { published++ }
+ layoutNudge.Store(&f)
+ nudgeLayoutLeg()
+ if published != 1 {
+ t.Fatalf("published kick ran %d times", published)
+ }
+}
+
+// The spawn record keeps one stretch per host: a count and the first and
+// last time, restarted when the reason changes.
+func TestFlatExecRecordStretches(t *testing.T) {
+ resetFlatExec(t)
+ armFlatExec(envOf(map[string]string{"CARLOS_FLAT_EXEC": "refuse"}), nil)
+ s := armedFlatExec.Load()
+ clock := time.Date(2026, 10, 25, 12, 0, 0, 0, time.UTC)
+ s.now = func() time.Time { return clock }
+ s.record("a.example.com", activator.FlatExecFlatRefused)
+ clock = clock.Add(time.Minute)
+ s.record("a.example.com", activator.FlatExecFlatRefused)
+ got := s.refusals()
+ if len(got) != 1 || got[0].Count != 2 || !got[0].Since.Equal(clock.Add(-time.Minute)) || !got[0].Last.Equal(clock) {
+ t.Fatalf("two refusals: %+v", got)
+ }
+ clock = clock.Add(time.Minute)
+ s.record("a.example.com", activator.FlatExecForeignRefused)
+ got = s.refusals()
+ if got[0].Reason != activator.FlatExecForeignRefused || got[0].Count != 1 || !got[0].Since.Equal(clock) {
+ t.Fatalf("a changed reason did not start a new stretch: %+v", got)
+ }
+ // The cap evicts the oldest record, never the newest.
+ for i := 0; i < flatExecRecordsCap+5; i++ {
+ clock = clock.Add(time.Second)
+ s.record(strings.Repeat("h", 1+i%3)+string(rune('a'+i%26))+".example.com"+strings.Repeat("x", i/26), activator.FlatExecFlatRefused)
+ }
+ if n := len(s.refusals()); n > flatExecRecordsCap {
+ t.Fatalf("records = %d, over the cap %d", n, flatExecRecordsCap)
+ }
+}
+
+// Layout.Workers (D14): the cached readiness word unless it holds, and then
+// nested only with a nested intent, an answering gate and claim mode off;
+// blocked, with the reason, otherwise.
+func TestWorkersStatusWord(t *testing.T) {
+ resetFlatExec(t)
+ resetLayoutGlobals(t)
+ noEnv := envOf(nil)
+ startWorkers.Store(nil)
+ if w, _ := workersStatusWord(noEnv); w != "off" {
+ t.Fatalf("unarmed = %q, want off", w)
+ }
+ startWorkers.Store(&WorkersReadiness{word: workersWaitingForPrev, reason: "prev below"})
+ if w, r := workersStatusWord(noEnv); w != "waiting_for_prev" || r != "prev below" {
+ t.Fatalf("waiting = %q %q", w, r)
+ }
+ startWorkers.Store(&WorkersReadiness{ready: true, word: workersNested})
+ if w, r := workersStatusWord(noEnv); w != "blocked" || !strings.Contains(r, "unknown") {
+ t.Fatalf("ready with no intent = %q %q, want blocked on the intent", w, r)
+ }
+ noteLayoutIntent(release.LayoutModeNested)
+ if w, r := workersStatusWord(noEnv); w != "blocked" || !strings.Contains(r, "no activator") {
+ t.Fatalf("ready, nested, no gate = %q %q", w, r)
+ }
+ gateErr := errors.New("mounts missing")
+ gate := func() error { return gateErr }
+ layoutDefaultGate.Store(&gate)
+ if w, r := workersStatusWord(noEnv); w != "blocked" || !strings.Contains(r, "mounts missing") {
+ t.Fatalf("ready, nested, failing gate = %q %q", w, r)
+ }
+ gateErr = nil
+ if w, r := workersStatusWord(envOf(map[string]string{"CARLOS_HOST_CLAIM": "on"})); w != "blocked" || !strings.Contains(r, "claim") {
+ t.Fatalf("claim mode = %q %q", w, r)
+ }
+ if w, r := workersStatusWord(noEnv); w != "nested" || r != "" {
+ t.Fatalf("everything holds = %q %q, want nested", w, r)
+ }
+ // And it rides on the layout report, whatever the pass state.
+ if rep := layoutStatusReport(nil); rep.Workers != "nested" {
+ t.Fatalf("layout report workers = %q", rep.Workers)
+ }
+ noteLayoutIntent(release.LayoutModeFlat)
+ if rep := layoutStatusReport(nil); rep.Workers != "blocked" || !strings.Contains(rep.WorkersReason, "flat_choice") {
+ t.Fatalf("layout report under flat = %q %q", rep.Workers, rep.WorkersReason)
+ }
+}
diff --git a/internal/host/layoutleg.go b/internal/host/layoutleg.go
index 8ccc4606c..a021b48fe 100644
--- a/internal/host/layoutleg.go
+++ b/internal/host/layoutleg.go
@@ -190,6 +190,16 @@ func noteLayoutDefault(mode, reason string) {
// this never lists the registry itself, mirroring every other probe in
// this file.
func layoutStatusReport(routes []registry.Route) *status.LayoutReport {
+ r := layoutRowsReport(routes)
+ // The workers readiness word (sidecar isolation D11, D14) rides on
+ // the layout report whatever its mode: it is about this agent, not
+ // about any row.
+ r.Workers, r.WorkersReason = workersStatusWord(os.Getenv)
+ return r
+}
+
+// layoutRowsReport is layoutStatusReport without the workers word.
+func layoutRowsReport(routes []registry.Route) *status.LayoutReport {
pass := armedLayoutPass.Load()
if pass == nil {
return unmanagedLayoutReport(routes)