amadan
units: carlos-instance-cmd@, the seed template outside six lines (run spec §5.4)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
7 files changed,
+236
−10
approved_run_copy_test.go+1 −1docs/site/cli.md+3 −3internal/host/host_test.go+1 −0internal/host/units/carlos-instance-cmd@.service+163 −0internal/host/units_hardening_test.go+4 −3internal/host/units_isolation_test.go+63 −2internal/instanceexec/families_test.go+1 −1
diff --git a/approved_run_copy_test.go b/approved_run_copy_test.go| index a92e2db90..d6a0ce00f 100644 |
| --- a/approved_run_copy_test.go |
| +++ b/approved_run_copy_test.go |
| @@ -33,7 +33,7 @@ var runCopyIDs = func() []string { |
| // runCopyBuilt is every deck id whose text the code or docs now carry. The |
| // task that writes a row's text adds its id here in the same commit. |
| -var runCopyBuilt = []string{"RUN-C9", "RUN-C10", "RUN-C13", "RUN-C14"} |
| +var runCopyBuilt = []string{"RUN-C9", "RUN-C10", "RUN-C11", "RUN-C12", "RUN-C13", "RUN-C14"} |
| func mustParseRunRecord(t *testing.T) []copyRow { |
| t.Helper() |
diff --git a/docs/site/cli.md b/docs/site/cli.md| index 44dc7d488..95bf4bc59 100644 |
| --- a/docs/site/cli.md |
| +++ b/docs/site/cli.md |
| @@ -2073,12 +2073,12 @@ link with exactly the arguments and environment in it, refusing anything |
| that is not the platform's own live link. It then stays on as the |
| instance's first process: it passes stop signals on to the app, cleans up |
| exited processes, and exits with the app's status, so an app with no |
| -signal handling still stops cleanly. `ExecStart=` of the three instance |
| -templates. |
| +signal handling still stops cleanly. |
| +`ExecStart=` of the four instance templates. |
| ### carlos instance-exit |
| -`ExecStopPost=` of the three instance templates. It leaves an advisory |
| +`ExecStopPost=` of the four instance templates. It leaves an advisory |
| exit record in the instance's run directory from the four variables |
| systemd provides; the agent proves termination from the unit's state and |
| its cgroup, not from this file. |
diff --git a/internal/host/host_test.go b/internal/host/host_test.go| index 4bd257c09..07d5bc59e 100644 |
| --- a/internal/host/host_test.go |
| +++ b/internal/host/host_test.go |
| @@ -39,6 +39,7 @@ var expectedUnits = []string{ |
| "carlos-edge.service", |
| "carlos-hibernate-verify.service", |
| "carlos-hibernate-verify.timer", |
| + "carlos-instance-cmd@.service", |
| "carlos-instance-control.service", |
| "carlos-instance-control.socket", |
| "carlos-instance-probe.service", |
diff --git a/internal/host/units/carlos-instance-cmd@.service b/internal/host/units/carlos-instance-cmd@.service| new file mode 100644 |
| index 000000000..c3237c770 |
| --- /dev/null |
| +++ b/internal/host/units/carlos-instance-cmd@.service |
| @@ -0,0 +1,163 @@ |
| +# One `carlos run` command against an isolated exec-backed instance (run |
| +# spec §5.4). It is the seed template outside six lines, and a test holds |
| +# it there (TestCmdTemplateEqualsSeedOutsideTheNamedLines): the same uid, |
| +# namespace, bind mounts, slice and working directory as the instance, so |
| +# the command has exactly the instance's reach. The agent starts it |
| +# through the control service's POST /cmd and follows it with GET |
| +# /cmd/<host>; the agent, not this file, enforces the run's own timeout. |
| +[Unit] |
| +Description=carlos exec-backed instance %I (cmd) |
| +# Stop-only coupling, as for the seed: when the edge stops, the command |
| +# stops, and only the agent starts one. A run picked before an agent |
| +# restart is never run again (run spec §5.2). |
| +BindsTo=carlos-edge.service |
| +Requires=data-instances.mount run-carlos-instances.mount run-carlos-logs.mount |
| +After=carlos-edge.service data-instances.mount run-carlos-instances.mount run-carlos-logs.mount |
| + |
| +[Service] |
| +Type=oneshot |
| +RemainAfterExit=yes |
| +# The result stays loaded, active (exited) or failed, until the agent has |
| +# read it and stopped the unit (codex 4). |
| +# A backstop past the run cap of 6 hours; the agent stops the unit at the |
| +# run's own timeout. |
| +TimeoutStartSec=6h5m |
| +ExecStart=/opt/carlos/carlos instance-exec cmd %I |
| +# The advisory exit record, as for the other templates. Not "+" (codex R3-2). |
| +ExecStopPost=/opt/carlos/carlos instance-exit cmd %I |
| +User=carlos |
| +Group=carlos |
| +Slice=carlos-instances-%i.slice |
| +# SIGTERM to the command, SIGKILL to anything left 20 s later. |
| +KillMode=mixed |
| +TimeoutStopSec=20 |
| +# No SuccessExitStatus=: the agent reads the exit status itself. |
| +# Two listeners, so stdout and stderr stay apart (run spec §5.5). PID 1 |
| +# opens both as root, outside the namespace, on the nosymfollow logs mount. |
| +StandardOutput=file:/run/carlos/logs/%I.cmd.sock |
| +StandardError=file:/run/carlos/logs/%I.cmd-err.sock |
| +# The instance's own two directories, the only writable data paths. |
| +BindPaths=/data/instances/%I /run/carlos/instances/%I |
| +ReadWritePaths=/data/instances/%I /run/carlos/instances/%I |
| +# The working directory is the instance's own data directory, the one its |
| +# database is in (discussion #256 item D). A flat child ran with cwd /, |
| +# and an app writing relative paths (./data/blobs) wrote under /data; |
| +# without this line a unit's cwd is / on a read-only tmpfs and that app |
| +# exits at start with "read-only file system". systemd applies it after |
| +# the namespace is set up, so the bind above is what it chdirs into. Same |
| +# value in all three templates: the seed and the sidecar see the files |
| +# the instance made from the same place. |
| +WorkingDirectory=/data/instances/%I |
| +# Its own build, and no other app's (#214 item 2). The namespace block |
| +# empties /opt/carlos; these put back /opt/carlos/live/<host>, a link to |
| +# ../app, and /opt/carlos/app, which PID 1 binds by chasing the source |
| +# link to this host's live build at every start. The control service |
| +# writes both sources before it starts the unit. |
| +BindReadOnlyPaths=/run/carlos-control/exe/%I/app:/opt/carlos/app /run/carlos-control/exe/%I/live:/opt/carlos/live |
| + |
| +# --- the namespace (titogo@.service, 2026-07-31 design, on systemd 257) --- |
| +ProtectSystem=strict |
| +TemporaryFileSystem=/data:ro |
| +TemporaryFileSystem=/run/carlos:ro |
| +# Every other tree a carlos-uid unit writes on the box is hidden the same |
| +# way (codex R9-2): the restore drills' cache under /var/cache, and |
| +# /var/lib as a class. An instance needs nothing from either. |
| +TemporaryFileSystem=/var/cache:ro |
| +TemporaryFileSystem=/var/lib:ro |
| +# The version store holds every app's build on the box, world-readable so |
| +# a unit can exec its own (#214 item 2). An instance sees none of it: an |
| +# empty /opt/carlos with the platform binary that ExecStart= and |
| +# ExecStopPost= run put back, and, outside this block because it is per |
| +# instance, its own build at /opt/carlos/live/<host>. |
| +TemporaryFileSystem=/opt/carlos:ro |
| +BindReadOnlyPaths=/opt/carlos/carlos |
| +# Root-written credentials a carlos-GROUP process may read live here: |
| +# /etc/carlos/relay-masters is root:carlos 0640 for carlos-relay.service, |
| +# and an instance's Group=carlos would read it too (codex R10-1). An |
| +# instance gets its configuration through its launch file and needs |
| +# nothing from /etc/carlos. /run/carlos-control is the root control |
| +# service's socket directory (design §5.2): hiding it here too means a |
| +# connect from inside any instance's namespace is impossible outright, |
| +# not merely refused by the peer check on the far end (final review, |
| +# plan 2, item 3). |
| +InaccessiblePaths=/etc/carlos /run/carlos-control |
| +ProtectHome=yes |
| +PrivateTmp=yes |
| +PrivateDevices=yes |
| +# PrivateDevices= bind-mounts the HOST's /dev/shm into the private /dev, |
| +# and every instance is uid carlos, so a sibling's shared memory would be |
| +# one open away. Each unit gets its own, charged to its own cgroup. |
| +TemporaryFileSystem=/dev/shm:mode=1777,nosuid,nodev |
| +# POSIX message queues and SysV IPC belong to an IPC namespace, and every |
| +# instance is uid carlos, so a sibling's queue would be one mq_open away |
| +# (#178). PrivateIPC= gives each unit its own namespace, and mq_open() |
| +# goes through that namespace's own internal mount. PrivateDevices= |
| +# carries the host's /dev/mqueue into the private /dev the way it carries |
| +# /dev/shm, and that mount would still list and open the host namespace's |
| +# queues, so it is replaced the same way. |
| +PrivateIPC=yes |
| +TemporaryFileSystem=/dev/mqueue:ro |
| +# /sys/fs/cgroup read-only: the edge's delegated subtree is carlos-owned, |
| +# and its cgroup.kill / cgroup.freeze would otherwise be one write away |
| +# for a same-uid tenant (codex R9-1). |
| +ProtectControlGroups=yes |
| +ProtectProc=invisible |
| +ProcSubset=pid |
| +PrivatePIDs=yes |
| +RestrictNamespaces=yes |
| +NoNewPrivileges=yes |
| +CapabilityBoundingSet= |
| +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX |
| +# Abstract unix sockets live in the network namespace, which instances |
| +# share with the box because they need outbound TCP, and the line above |
| +# cannot tell them from path sockets (#178). The trampoline scopes them |
| +# with Landlock before it execs the app (instanceexec.ScopeAbstractUnix): |
| +# an instance can connect only to abstract sockets its own processes |
| +# bound. A sibling can still bind a name first; that is a refusal of |
| +# service, never a connection. |
| +# |
| +# Egress (#214 item 3): the internet, and nothing on the box or private. |
| +# Loopback is where the console and other tenants' addr upstreams listen; |
| +# RFC 1918, ULA, link-local and CGNAT are the VPC, other boxes and the |
| +# cloud's internal services. IMDS stays named so the property is |
| +# greppable. A cgroup filter: it drops in both directions for every |
| +# socket the unit opens. The one thing an instance needs from these |
| +# ranges is the box's DNS resolver, so the control service re-opens |
| +# exactly the nameservers the instance sees with IPAddressAllow= on |
| +# carlos-instances.slice (an allow wins, and a slice's lists reach every |
| +# unit below it). The probe checks both halves. |
| +IPAddressDeny=localhost multicast 0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 169.254.0.0/16 169.254.169.254/32 172.16.0.0/12 192.0.0.0/24 192.168.0.0/16 198.18.0.0/15 240.0.0.0/4 64:ff9b::/96 fc00::/7 fd00:ec2::254/128 fe80::/10 fec0::/10 |
| +# An allow opens an address both ways and on every port, so a loopback |
| +# resolver can never be opened: a tenant could bind systemd-resolved's |
| +# 127.0.0.53 itself and reach every listener bound to all addresses |
| +# through it. Where resolved runs, instances resolve through its |
| +# upstreams instead, and the control service opens those. The "-" makes |
| +# this a no-op on a box without resolved. |
| +BindReadOnlyPaths=-/run/systemd/resolve/resolv.conf:/etc/resolv.conf |
| +ProtectKernelTunables=yes |
| +ProtectKernelModules=yes |
| +ProtectKernelLogs=yes |
| +ProtectClock=yes |
| +ProtectHostname=yes |
| +RemoveIPC=yes |
| +LockPersonality=yes |
| +RestrictRealtime=yes |
| +# No RestrictSUIDSGID=: systemd cannot read the mode inside openat2's |
| +# open_how, so it refuses openat2 outright (ENOSYS), and every instancefs |
| +# open in here is an openat2 (#126, first roll; systemd 257). What it |
| +# guarded is covered: with no capabilities an instance can only make a |
| +# set-id file owned by carlos, which gains nothing for another carlos |
| +# process, NoNewPrivileges= voids it in here, and no other uid can reach |
| +# one (the tenant trees are nosuid, PrivateTmp's sit under a root 0700 |
| +# directory). |
| +MemoryDenyWriteExecute=yes |
| +SystemCallArchitectures=native |
| + |
| +# --- per-service resources; the tier lives on the host slice (§4.3) --- |
| +TasksMax=256 |
| +# The kernel picks a tenant before it picks anything else in carlos.slice. |
| +OOMScoreAdjust=500 |
| +# One OOM kill takes every process of THIS service (systemd sets |
| +# memory.oom.group on the service cgroup for OOMPolicy=kill). The sidecar |
| +# is a sibling service with its own family rule (§4.3). |
| +OOMPolicy=kill |
diff --git a/internal/host/units_hardening_test.go b/internal/host/units_hardening_test.go| index 0cd6fc864..513c3e954 100644 |
| --- a/internal/host/units_hardening_test.go |
| +++ b/internal/host/units_hardening_test.go |
| @@ -39,10 +39,10 @@ const instanceDeny = "IPAddressDeny=localhost multicast " + |
| // Every directive the hardening adds to the shared namespace block (#214 |
| // items 2 and 3, #178), pinned exactly: the block is byte-identical across |
| -// the three templates and the probe (TestInstanceTemplatesShareOneNamespace), |
| -// so pinning it on carlos-instance@ pins it on all four. |
| +// the four templates and the probe (TestInstanceTemplatesShareOneNamespace), |
| +// so pinning it on carlos-instance@ pins it on all five. |
| func TestNamespaceBlockCarriesTheHardening(t *testing.T) { |
| - for _, name := range []string{"carlos-instance@.service", "carlos-instance-seed@.service", "carlos-instance-sidecar@.service", "carlos-instance-probe.service"} { |
| + for _, name := range []string{"carlos-instance@.service", "carlos-instance-seed@.service", "carlos-instance-sidecar@.service", "carlos-instance-cmd@.service", "carlos-instance-probe.service"} { |
| u := readUnit(t, name) |
| for _, want := range []string{ |
| // #214 item 2: no other app's build is visible. |
| @@ -108,6 +108,7 @@ func TestInstanceTemplatesBindOnlyTheirOwnBuild(t *testing.T) { |
| "carlos-instance@.service": "%I", |
| "carlos-instance-seed@.service": "%I", |
| "carlos-instance-sidecar@.service": "%I", |
| + "carlos-instance-cmd@.service": "%I", |
| "carlos-instance-probe.service": "_probe", |
| } { |
| u := readUnit(t, name) |
diff --git a/internal/host/units_isolation_test.go b/internal/host/units_isolation_test.go| index f7a939f57..def5b5825 100644 |
| --- a/internal/host/units_isolation_test.go |
| +++ b/internal/host/units_isolation_test.go |
| @@ -239,7 +239,7 @@ func TestInstanceTemplatesShareOneNamespace(t *testing.T) { |
| return u[start:end] |
| } |
| ref := block("carlos-instance@.service") |
| - for _, name := range []string{"carlos-instance-seed@.service", "carlos-instance-sidecar@.service", "carlos-instance-probe.service"} { |
| + for _, name := range []string{"carlos-instance-seed@.service", "carlos-instance-sidecar@.service", "carlos-instance-cmd@.service", "carlos-instance-probe.service"} { |
| if block(name) != ref { |
| t.Errorf("%s namespace block differs from carlos-instance@.service", name) |
| } |
| @@ -278,6 +278,9 @@ func TestInstanceTemplatesShareOneNamespace(t *testing.T) { |
| "carlos-instance-seed@.service": {"Type=oneshot", "RemainAfterExit=yes", "TimeoutStartSec=120", "ExecStart=/opt/carlos/carlos instance-exec seed %I", |
| "ExecStopPost=/opt/carlos/carlos instance-exit seed %I", "StandardOutput=file:/run/carlos/logs/%I.seed.sock", |
| "KillMode=mixed", "TimeoutStopSec=20", "WorkingDirectory=/data/instances/%I"}, |
| + "carlos-instance-cmd@.service": {"Type=oneshot", "RemainAfterExit=yes", "TimeoutStartSec=6h5m", "ExecStart=/opt/carlos/carlos instance-exec cmd %I", |
| + "ExecStopPost=/opt/carlos/carlos instance-exit cmd %I", "StandardOutput=file:/run/carlos/logs/%I.cmd.sock", |
| + "StandardError=file:/run/carlos/logs/%I.cmd-err.sock", "KillMode=mixed", "TimeoutStopSec=20", "WorkingDirectory=/data/instances/%I"}, |
| "carlos-instance-sidecar@.service": {"Type=simple", "ExecStart=/opt/carlos/carlos instance-exec sidecar %I", |
| "ExecStopPost=/opt/carlos/carlos instance-exit sidecar %I", "StandardOutput=file:/run/carlos/logs/%I.sidecar.sock", |
| "SuccessExitStatus=129 130 141 143", "WorkingDirectory=/data/instances/%I"}, |
| @@ -300,7 +303,7 @@ func TestInstanceTemplatesShareOneNamespace(t *testing.T) { |
| t.Errorf("%s uses ExecStopPost=+: the plus prefix keeps the unit's tmpfs (codex R3-2)", name) |
| } |
| // A seed a signal ended has not seeded (#126). |
| - if name == "carlos-instance-seed@.service" && strings.Contains(u, "\nSuccessExitStatus=") { |
| + if (name == "carlos-instance-seed@.service" || name == "carlos-instance-cmd@.service") && strings.Contains(u, "\nSuccessExitStatus=") { |
| t.Errorf("%s counts a signal's exit status as success", name) |
| } |
| if name != "carlos-instance-probe.service" && !strings.Contains(u, "User=carlos") { |
| @@ -405,3 +408,61 @@ func TestSameUIDAuditIsCoveredByTheTemplates(t *testing.T) { |
| } |
| } |
| } |
| + |
| +// The cmd template is the seed template outside six named lines (run spec |
| +// §5.4, codex 4): every sandbox line is the seed's, so a command has |
| +// exactly the reach a seed has. Comments and blank lines are free; every |
| +// directive is compared, in order. |
| +func TestCmdTemplateEqualsSeedOutsideTheNamedLines(t *testing.T) { |
| + directives := func(name string) []string { |
| + var out []string |
| + for _, line := range strings.Split(readUnit(t, name), "\n") { |
| + line = strings.TrimSpace(line) |
| + if line == "" || strings.HasPrefix(line, "#") { |
| + continue |
| + } |
| + out = append(out, line) |
| + } |
| + return out |
| + } |
| + swap := map[string]string{ |
| + "Description=carlos exec-backed instance %I (seed)": "Description=carlos exec-backed instance %I (cmd)", |
| + "TimeoutStartSec=120": "TimeoutStartSec=6h5m", |
| + "ExecStart=/opt/carlos/carlos instance-exec seed %I": "ExecStart=/opt/carlos/carlos instance-exec cmd %I", |
| + "ExecStopPost=/opt/carlos/carlos instance-exit seed %I": "ExecStopPost=/opt/carlos/carlos instance-exit cmd %I", |
| + "StandardOutput=file:/run/carlos/logs/%I.seed.sock": "StandardOutput=file:/run/carlos/logs/%I.cmd.sock", |
| + "StandardError=inherit": "StandardError=file:/run/carlos/logs/%I.cmd-err.sock", |
| + } |
| + seed, cmd := directives("carlos-instance-seed@.service"), directives("carlos-instance-cmd@.service") |
| + want := make([]string, len(seed)) |
| + used := map[string]bool{} |
| + for i, l := range seed { |
| + if c, ok := swap[l]; ok { |
| + want[i], used[l] = c, true |
| + continue |
| + } |
| + want[i] = l |
| + } |
| + for l := range swap { |
| + if !used[l] { |
| + t.Errorf("the seed template no longer has %q; the run spec names it as a cmd difference", l) |
| + } |
| + } |
| + for i := 0; i < len(want) || i < len(cmd); i++ { |
| + var w, g string |
| + if i < len(want) { |
| + w = want[i] |
| + } |
| + if i < len(cmd) { |
| + g = cmd[i] |
| + } |
| + if w != g { |
| + t.Fatalf("directive %d: cmd template has %q, want %q (the seed's line with the named swaps)", i, g, w) |
| + } |
| + } |
| + for _, keep := range []string{"Type=oneshot", "RemainAfterExit=yes", "KillMode=mixed", "TimeoutStopSec=20", "WorkingDirectory=/data/instances/%I"} { |
| + if !hasDirective(readUnit(t, "carlos-instance-cmd@.service"), keep) { |
| + t.Errorf("cmd template lacks %q", keep) |
| + } |
| + } |
| +} |
diff --git a/internal/instanceexec/families_test.go b/internal/instanceexec/families_test.go| index 937b5c945..08b322026 100644 |
| --- a/internal/instanceexec/families_test.go |
| +++ b/internal/instanceexec/families_test.go |
| @@ -15,7 +15,7 @@ import ( |
| ) |
| // Every carlos process inside an instance unit (the probe, the trampoline |
| -// and its exit hook, for all three templates) runs under |
| +// and its exit hook, for all four templates) runs under |
| // RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX. A socket of any other |
| // family is refused, so code that asks for one fails on every box: the |
| // probe's net.InterfaceAddrs, which is netlink, kept every box on v0.25.0 |