67e40b9a2 · carlos/platform

diff --git a/internal/host/bootstrap.go b/internal/host/bootstrap.go

index 0b258385d..bc9175c0d 100644

--- a/internal/host/bootstrap.go

+++ b/internal/host/bootstrap.go

@@ -30,7 +30,7 @@ import (

// no separate files to copy. Ported from Eleven's deploy/systemd (renamed,

// same hardening) and titogo's deploy/systemd (litestream + restore-verify).

//

-//go:embed units/*.service units/*.timer units/*.socket units/*.path

+//go:embed units/*.service units/*.timer units/*.socket units/*.path units/*.slice units/*.mount

var unitsFS embed.FS

// serviceUser is the unprivileged account the edge unit runs as; bootstrap

diff --git a/internal/host/host_test.go b/internal/host/host_test.go

index 1a5b09dbd..9dbc0a1d7 100644

--- a/internal/host/host_test.go

+++ b/internal/host/host_test.go

@@ -36,6 +36,7 @@ var expectedUnits = []string{

"carlos-edge.service",

"carlos-hibernate-verify.service",

"carlos-hibernate-verify.timer",

+ "carlos-instances.slice",

"carlos-litestream.service",

"carlos-relay.service",

"carlos-restart.path",

@@ -47,6 +48,10 @@ var expectedUnits = []string{

"carlos-system-recover.service",

"carlos-system-update.path",

"carlos-system-update.service",

+ "carlos.slice",

+ "data-instances.mount",

+ "run-carlos-instances.mount",

+ "run-carlos-logs.mount",

}

// requireUnixSockets skips a test when the sandbox forbids unix-socket

@@ -178,19 +183,34 @@ func TestBootstrapWritesUnitsToStagingDir(t *testing.T) {

// cgroup manager degrades to nil and every exec-backed tenant goes

// blind on the Instances page.

"Delegate=cpu memory io pids",

- // The outer wall (memory-pressure spec §2.2): the edge tree is

- // where every exec-backed tenant lives, so this is the one place a

- // runaway tenant meets a ceiling before the kernel starts thrashing.

- // OOMPolicy=continue is the line that keeps one tenant's OOM kill

- // from becoming a whole-edge restart (systemd's default is stop).

- "MemoryHigh=70%",

- "MemoryMax=80%",

+ // The outer wall (memory-pressure spec §2.2) is now on carlos.slice

+ // (isolated-exec design §4.3). The edge is in the slice, so it still

+ // participates in the ceiling. OOMPolicy=continue is the line that

+ // keeps one tenant's OOM kill from becoming a whole-edge restart

+ // (systemd's default is stop).

+ "Slice=carlos.slice",

"OOMPolicy=continue",

} {

if !strings.Contains(string(edge), want) {

t.Errorf("carlos-edge.service: missing %q", want)

}

+

+ // The outer wall moved from the edge to carlos.slice (isolated-exec

+ // design §4.3): the slice bounds the edge and every awake instance

+ // as before, but instances are now sibling units instead of children.

+ slice, err := os.ReadFile(filepath.Join(unitDir, "carlos.slice"))

+ if err != nil {

+ t.Fatal(err)

+ }

+ for _, want := range []string{

+ "MemoryHigh=70%",

+ "MemoryMax=80%",

+ } {

+ if !strings.Contains(string(slice), want) {

+ t.Errorf("carlos.slice: missing %q", want)

+ }

// The counterpart of Delegate=: ProtectControlGroups would mount the

// cgroup tree read-only and silently un-arm delegation. Its hardening is

// retained by ownership (DAC) instead — see the unit file's comment. A

diff --git a/internal/host/units/carlos-edge.service b/internal/host/units/carlos-edge.service

index 018304cec..c988f2af8 100644

--- a/internal/host/units/carlos-edge.service

+++ b/internal/host/units/carlos-edge.service

@@ -5,7 +5,11 @@

# managed instances get their own units.

[Unit]

Description=carlos edge (TLS terminator, host router, activator, release adoption)

-After=network-online.target

+# The three nosymfollow mounts of the isolated-exec design (§3.2) are hard

+# requirements: an edge that starts without them would start unprotected,

+# so it does not start at all.

+Requires=data-instances.mount run-carlos-instances.mount run-carlos-logs.mount

+After=network-online.target data-instances.mount run-carlos-instances.mount run-carlos-logs.mount

Wants=network-online.target

[Service]

@@ -54,22 +58,13 @@ MemoryDenyWriteExecute=yes

SystemCallFilter=@system-service

SystemCallErrorNumber=EPERM

SystemCallArchitectures=native

-# The outer wall (memory-pressure spec 2026-08-28 §2.2). Exec-backed

-# instances are this unit's children, so these DO cap the whole set of

-# awake instances at once — that is now the point: on 2026-08-28 the

-# flagship thrashed for 15 minutes with no swap, no ceiling and an OOM

-# killer that never fired, because nothing bounded the tree. Percentages

-# so one file fits a t4g.micro and a t4g.medium; 80% leaves what the

-# box's other units (keymail, console, ssm, litestream, journald) use.

-# Past High the kernel reclaims the tree (page cache first, then zram);

-# at Max the memcg OOM killer picks the highest-scoring PROCESS in the

-# tree. By RSS alone that would be the agent itself (~110 MB against

-# 20-60 MB tenant processes), so the agent writes oom_score_adj=+500 on

-# every exec child it starts (internal/cgroup ExecOOMScoreAdj) — a tenant

-# process is always the first pick. Per-instance caps below this

-# (carlos-exec@<host> memory.max, internal/cgroup) are the inner walls.

-MemoryHigh=70%

-MemoryMax=80%

+# The outer wall (memory-pressure spec 2026-08-28 §2.2) used to be two

+# MemoryHigh/MemoryMax lines here, bounding the agent plus its exec

+# children. Instances are sibling units now (isolated-exec design §4.3), so

+# the wall is carlos.slice's, which holds this unit and every instance and

+# bounds exactly what it bounded before. The oom_score_adj story is

+# unchanged: tenants carry OOMScoreAdjust=500 in their template.

+Slice=carlos.slice

# systemd's default OOMPolicy=stop STOPS the whole service when the kernel

# OOM-kills ANY process in the unit's cgroup tree (memory.events is

# hierarchical, delegation or not). Restart=always would bring the edge

diff --git a/internal/host/units/carlos-instances.slice b/internal/host/units/carlos-instances.slice

new file mode 100644

index 000000000..2930ac8bc

--- /dev/null

+++ b/internal/host/units/carlos-instances.slice

@@ -0,0 +1,10 @@

+# The parent of every per-host slice (carlos-instances-<iid>.slice, created

+# on demand by the first carlos-instance*@ unit that names it; isolated-exec

+# design §4.3). No cap of its own: the outer wall is carlos.slice, the inner

+# walls are the per-host slices the control service sets the tier on. This

+# level exists so the sampler has one root to scan and so an operator can

+# `systemctl status carlos-instances.slice`.

+[Unit]

+Description=carlos exec-backed instances

+

+[Slice]

diff --git a/internal/host/units/carlos-litestream.service b/internal/host/units/carlos-litestream.service

index dbc923f8e..edb17ddf5 100644

--- a/internal/host/units/carlos-litestream.service

+++ b/internal/host/units/carlos-litestream.service

@@ -9,7 +9,7 @@

[Unit]

Description=carlos litestream (replicate /data SQLite files to S3)

Documentation=https://litestream.io

-After=network-online.target

+After=network-online.target data-instances.mount

Wants=network-online.target

[Service]

diff --git a/internal/host/units/carlos.slice b/internal/host/units/carlos.slice

new file mode 100644

index 000000000..6f78ac1a9

--- /dev/null

+++ b/internal/host/units/carlos.slice

@@ -0,0 +1,18 @@

+# The outer memory wall of the memory-pressure containment design

+# (2026-08-28 §2.2), moved here from carlos-edge.service by the isolated-exec

+# design (§4.3): exec-backed instances used to be the edge's children, so a

+# ceiling on the edge unit bounded the agent plus every awake instance at

+# once. They are sibling units now, so the ceiling lives one level up, on

+# a slice that holds the edge AND carlos-instances.slice, and bounds exactly

+# what it bounded before. Borrowing between agent and tenants and the OOM

+# candidate set (a tenant at OOMScoreAdjust=500 is picked before the agent)

+# are unchanged.

+#

+# Percentages so one file fits a t4g.micro and a t4g.medium; 80% leaves what

+# the box's other units (keymail, console, ssm, litestream, journald) use.

+[Unit]

+Description=carlos (edge, activator and every exec-backed instance)

+

+[Slice]

+MemoryHigh=70%

+MemoryMax=80%

diff --git a/internal/host/units/data-instances.mount b/internal/host/units/data-instances.mount

new file mode 100644

index 000000000..b2ce7f094

--- /dev/null

+++ b/internal/host/units/data-instances.mount

@@ -0,0 +1,38 @@

+# The protected view of the instance data tree (isolated-exec design §3.2).

+#

+# /data/.instances is the RAW directory, opened by the layout pass only;

+# this bind of it onto /data/instances is the view everything else uses.

+# nosymfollow is the isolation's belt: path resolution under this mount

+# returns ELOOP at any symlink, for every process on the box, in every

+# namespace that sees the mount, with no check-then-use race. An instance

+# runs as the agent's uid and can write its own directory, so any symlink

+# it plants would otherwise be followed by the agent (reading a sibling's

+# env file as this instance's), by the edge (dialling a sibling's socket

+# as this host) and by PID 1. Here none of them follow it.

+#

+# It is a bind of a DIFFERENT directory, not of itself, because rename(2)

+# refuses to cross a mount boundary (EXDEV) even on one filesystem, and

+# the migration moves /data/<host>.db into this tree by rename (codex R4-2).

+#

+# DefaultDependencies=no, with the ordering written out: a mount unit's

+# default place is before local-fs.target, and systemd-tmpfiles-setup,

+# which creates the two directories, runs after it, so the defaults form a

+# cycle systemd breaks by dropping this mount (codex R2-2). Shutdown

+# ordering is written out for the same reason.

+[Unit]

+Description=carlos instance data (bind, nosymfollow)

+DefaultDependencies=no

+RequiresMountsFor=/data

+After=local-fs.target systemd-tmpfiles-setup.service

+Before=carlos-edge.service carlos-instance-control.service carlos-litestream.service

+Conflicts=umount.target

+Before=umount.target

+

+[Mount]

+What=/data/.instances

+Where=/data/instances

+Type=none

+Options=bind,nosymfollow,nosuid,nodev,noexec

+

+[Install]

+WantedBy=multi-user.target

diff --git a/internal/host/units/run-carlos-instances.mount b/internal/host/units/run-carlos-instances.mount

new file mode 100644

index 000000000..9de118149

--- /dev/null

+++ b/internal/host/units/run-carlos-instances.mount

@@ -0,0 +1,23 @@

+# The protected view of the instance socket tree (isolated-exec design

+# §3.2): /run/carlos/.instances bound onto /run/carlos/instances with

+# nosymfollow. See data-instances.mount for why a bind, why of a different

+# directory, and why DefaultDependencies=no. Nothing is ever renamed into

+# this tree, so the raw directory has no caller at all. /run is tmpfs, so

+# the two directories come from the tmpfiles line every boot, which is why

+# this is ordered after systemd-tmpfiles-setup.service.

+[Unit]

+Description=carlos instance sockets (bind, nosymfollow)

+DefaultDependencies=no

+After=systemd-tmpfiles-setup.service

+Before=carlos-edge.service carlos-instance-control.service

+Conflicts=umount.target

+Before=umount.target

+

+[Mount]

+What=/run/carlos/.instances

+Where=/run/carlos/instances

+Type=none

+Options=bind,nosymfollow,nosuid,nodev,noexec

+

+[Install]

+WantedBy=multi-user.target

diff --git a/internal/host/units/run-carlos-logs.mount b/internal/host/units/run-carlos-logs.mount

new file mode 100644

index 000000000..8f605ee24

--- /dev/null

+++ b/internal/host/units/run-carlos-logs.mount

@@ -0,0 +1,23 @@

+# The protected view of the agent's per-role log sockets (isolated-exec

+# design §3.2, §6.4): /run/carlos/.logs bound onto /run/carlos/logs with

+# nosymfollow. This directory is bound into NO instance namespace; the

+# mount option is for the transition release, in which unisolated flat

+# tenants still share the agent's uid and could otherwise turn a log

+# socket path into a symlink that PID 1 opens as root for some unit's

+# stdout (codex R3-1). See data-instances.mount for the ordering.

+[Unit]

+Description=carlos instance log sockets (bind, nosymfollow)

+DefaultDependencies=no

+After=systemd-tmpfiles-setup.service

+Before=carlos-edge.service carlos-instance-control.service

+Conflicts=umount.target

+Before=umount.target

+

+[Mount]

+What=/run/carlos/.logs

+Where=/run/carlos/logs

+Type=none

+Options=bind,nosymfollow,nosuid,nodev,noexec

+

+[Install]

+WantedBy=multi-user.target

diff --git a/internal/host/units_isolation_test.go b/internal/host/units_isolation_test.go

new file mode 100644

index 000000000..27a770336

--- /dev/null

+++ b/internal/host/units_isolation_test.go

@@ -0,0 +1,118 @@

+package host

+

+import (

+ "strings"

+ "testing"

+)

+

+func readUnit(t *testing.T, name string) string {

+ t.Helper()

+ b, err := unitsFS.ReadFile("units/" + name)

+ if err != nil {

+ t.Fatalf("%s: %v", name, err)

+ }

+ return string(b)

+}

+

+// The outer memory wall moves from the edge unit to carlos.slice

+// (isolated-exec design §4.3): it bounds the agent plus every awake

+// instance as before, but instances are now sibling units, not children.

+func TestCarlosSliceCarriesTheOuterWall(t *testing.T) {

+ s := readUnit(t, "carlos.slice")

+ for _, want := range []string{"[Slice]", "MemoryHigh=70%", "MemoryMax=80%"} {

+ if !strings.Contains(s, want) {

+ t.Errorf("carlos.slice lacks %q", want)

+ }

+ e := readUnit(t, "carlos-edge.service")

+ if !strings.Contains(e, "Slice=carlos.slice") {

+ t.Error("carlos-edge.service is not in carlos.slice")

+ }

+ for _, never := range []string{"MemoryHigh=", "MemoryMax="} {

+ if strings.Contains(e, never) {

+ t.Errorf("carlos-edge.service still carries %q; the wall is the slice's now", never)

+ }

+ // What must NOT change on the edge unit.

+ for _, keep := range []string{"OOMPolicy=continue", "Delegate=cpu memory io pids", "User=carlos",

+ "AmbientCapabilities=CAP_NET_BIND_SERVICE", "ReadWritePaths=/data /run/carlos", "RestrictNamespaces=yes"} {

+ if !strings.Contains(e, keep) {

+ t.Errorf("carlos-edge.service lost %q", keep)

+ }

+ i := readUnit(t, "carlos-instances.slice")

+ if !strings.Contains(i, "[Slice]") {

+ t.Error("carlos-instances.slice has no [Slice] section")

+ }

+ for _, never := range []string{"MemoryHigh=", "MemoryMax="} {

+ if strings.Contains(i, never) {

+ t.Errorf("carlos-instances.slice carries %q; the walls are carlos.slice and the per-host slices", never)

+ }

+}

+

+// The three mounts: a bind of a raw directory onto its protected view,

+// nosymfollow, with DefaultDependencies=no and explicit ordering, because

+// the default ordering (before local-fs.target) and tmpfiles (after it)

+// would form a cycle systemd breaks by dropping the mount (codex R2-2).

+func TestInstanceMountsAreShapedForNosymfollow(t *testing.T) {

+ cases := map[string][2]string{

+ "data-instances.mount": {"/data/.instances", "/data/instances"},

+ "run-carlos-instances.mount": {"/run/carlos/.instances", "/run/carlos/instances"},

+ "run-carlos-logs.mount": {"/run/carlos/.logs", "/run/carlos/logs"},

+ }

+ for name, ww := range cases {

+ m := readUnit(t, name)

+ for _, want := range []string{

+ "What=" + ww[0], "Where=" + ww[1], "Type=none",

+ "Options=bind,nosymfollow,nosuid,nodev,noexec",

+ "DefaultDependencies=no",

+ "After=", "systemd-tmpfiles-setup.service",

+ "Before=", "carlos-edge.service",

+ "Conflicts=umount.target", "Before=umount.target",

+ "WantedBy=multi-user.target",

+ } {

+ if !strings.Contains(m, want) {

+ t.Errorf("%s lacks %q", name, want)

+ }

+ if ww[0] == ww[1] {

+ t.Errorf("%s binds a directory onto itself; the migration rename would fail with EXDEV (codex R4-2)", name)

+ }

+ d := readUnit(t, "data-instances.mount")

+ for _, want := range []string{"RequiresMountsFor=/data", "local-fs.target", "carlos-litestream.service"} {

+ if !strings.Contains(d, want) {

+ t.Errorf("data-instances.mount lacks %q", want)

+ }

+ e := readUnit(t, "carlos-edge.service")

+ for _, want := range []string{"Requires=data-instances.mount run-carlos-instances.mount run-carlos-logs.mount",

+ "After=network-online.target data-instances.mount run-carlos-instances.mount run-carlos-logs.mount"} {

+ if !strings.Contains(e, want) {

+ t.Errorf("carlos-edge.service lacks %q: an edge that starts without the mounts must not start", want)

+ }

+ l := readUnit(t, "carlos-litestream.service")

+ if !strings.Contains(l, "After=network-online.target data-instances.mount") {

+ t.Error("carlos-litestream.service is not ordered after data-instances.mount")

+ }

+}

+

+// Every unit file, whatever its suffix, is embedded: a .slice or .mount

+// left out of the glob would be silently absent from every box.

+func TestEmbeddedUnitsIncludeSlicesAndMounts(t *testing.T) {

+ names, err := unitNames()

+ if err != nil {

+ t.Fatal(err)

+ }

+ have := map[string]bool{}

+ for _, n := range names {

+ have[n] = true

+ }

+ for _, want := range []string{"carlos.slice", "carlos-instances.slice", "data-instances.mount", "run-carlos-instances.mount", "run-carlos-logs.mount"} {

+ if !have[want] {

+ t.Errorf("%s is not embedded (bootstrap.go's go:embed glob)", want)

+ }

+}

carlos / platform Public

units: carlos.slice outer wall and the three nosymfollow instance mounts (plan 1, task 4)