amadan
units: rootfs templates and probe with the restated namespace block (rootfs apps plan 2, task 4)
Five units for an app shipped as an image: carlos-instance-rootfs@, -rootfs-gen@, -rootfs-worker@, -rootfs-cmd@ and the rootfs probe. Each starts from its binary sibling and carries one restated namespace block, byte for byte, written from the plan's line-by-line table (spec §5.3): RootImageOptions=ro,nosuid,nodev and MountAPIVFS=yes for the root itself; a private tmpfs on /tmp for PrivateTmp=; ProtectControlGroups=strict for =yes; TemporaryFileSystem=/dev/pts:ro (deviation 8); the /var tmpfs lines and InaccessiblePaths= dropped because the host's /var, /etc/carlos and /run/carlos-control are not in an image root; resolved's resolv.conf bind replaced outside the block by root's per-instance copy. IPAddressDeny= and MemoryDenyWriteExecute=yes stay (R11: a jit release loses it only through Task 5's drop-in). No app or live bind: the image is the app, mounted through root's exe layout for the unit (C3). rootfs@ and rootfs-cmd@ carry Slice=carlos-instances-%i.slice; the gen and worker templates key nothing by their instance string (slice, binds, image, log socket come from the per-start drop-in); the probe sits in carlos-instances.slice and also binds root's box-addrs list (#367; the binary probe's bind is Task 7a's). Tests pin the shared block, the exact diff against the binary block, the bind-source allowlist and the per-unit lines; expectedUnits and the must-not-be-enabled list learn the five names.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
8 files changed,
+1460
−1
internal/host/host_test.go+5 −0internal/host/units/carlos-instance-rootfs-cmd@.service+259 −0internal/host/units/carlos-instance-rootfs-gen@.service+239 −0internal/host/units/carlos-instance-rootfs-probe.service+231 −0internal/host/units/carlos-instance-rootfs-worker@.service+251 −0internal/host/units/carlos-instance-rootfs@.service+281 −0internal/host/units_isolation_test.go+2 −1internal/host/units_rootfs_test.go+192 −0
diff --git a/internal/host/host_test.go b/internal/host/host_test.go| index e1154574e..56794f00b 100644 |
| --- a/internal/host/host_test.go |
| +++ b/internal/host/host_test.go |
| @@ -47,6 +47,11 @@ var expectedUnits = []string{ |
| "carlos-instance-control.socket", |
| "carlos-instance-gen@.service", |
| "carlos-instance-probe.service", |
| + "carlos-instance-rootfs-cmd@.service", |
| + "carlos-instance-rootfs-gen@.service", |
| + "carlos-instance-rootfs-probe.service", |
| + "carlos-instance-rootfs-worker@.service", |
| + "carlos-instance-rootfs@.service", |
| "carlos-instance-seed@.service", |
| "carlos-instance-sidecar@.service", |
| "carlos-instance@.service", |
diff --git a/internal/host/units/carlos-instance-rootfs-cmd@.service b/internal/host/units/carlos-instance-rootfs-cmd@.service| new file mode 100644 |
| index 000000000..02ebbd06e |
| --- /dev/null |
| +++ b/internal/host/units/carlos-instance-rootfs-cmd@.service |
| @@ -0,0 +1,259 @@ |
| +# One `carlos run` command against a rootfs instance (run spec §5.4; |
| +# rootfs apps design §5.5): carlos-instance-cmd@.service for an app |
| +# shipped as an image. It is the rootfs instance template outside its |
| +# named lines: the same uid, namespace, image, bind mounts, slice and |
| +# working directory as the instance, so the command has exactly the |
| +# instance's reach. The agent starts it through the control service's |
| +# POST /cmd and follows it with GET /cmd/<host>; the agent, not this file, |
| +# enforces the run's own timeout. |
| +[Unit] |
| +Description=carlos rootfs instance %I (cmd) |
| +# Stop-only coupling, as for the instance: when the edge stops, the |
| +# command stops, and only the agent starts one. A run picked before an |
| +# agent restart is never run again (run spec §5.2). |
| +BindsTo=carlos-edge.service |
| +Requires=data-instances.mount run-carlos-instances.mount run-carlos-logs.mount |
| +After=carlos-edge.service data-instances.mount run-carlos-instances.mount run-carlos-logs.mount |
| + |
| +[Service] |
| +Type=oneshot |
| +RemainAfterExit=yes |
| +# The result stays loaded, active (exited) or failed, until the agent has |
| +# read it and stopped the unit (codex 4). |
| +# A backstop past the run cap of 6 hours; the agent stops the unit at the |
| +# run's own timeout. |
| +TimeoutStartSec=6h5m |
| +# --rootfs before the role, in ExecStart= and ExecStopPost= alike, as in |
| +# carlos-instance-rootfs@.service. |
| +ExecStart=/opt/carlos/carlos instance-exec --rootfs cmd %I |
| +# The advisory exit record, as for the other templates. Not "+" (codex R3-2). |
| +ExecStopPost=/opt/carlos/carlos instance-exit --rootfs cmd %I |
| +User=carlos |
| +Group=carlos |
| +Slice=carlos-instances-%i.slice |
| +# SIGTERM to the command, SIGKILL to anything left 20 s later. |
| +KillMode=mixed |
| +TimeoutStopSec=20 |
| +# No SuccessExitStatus=: the agent reads the exit status itself. |
| +# Two listeners, so stdout and stderr stay apart (run spec §5.5). PID 1 |
| +# opens both as root, outside the namespace, on the nosymfollow logs mount. |
| +StandardOutput=file:/run/carlos/logs/%I.cmd.sock |
| +StandardError=file:/run/carlos/logs/%I.cmd-err.sock |
| +# The instance's own two directories, the only writable data paths, bound |
| +# into the image root at the mount points ship adds (rootfs apps design |
| +# §5.1). |
| +BindPaths=/data/instances/%I /run/carlos/instances/%I |
| +ReadWritePaths=/data/instances/%I /run/carlos/instances/%I |
| +# The working directory is the instance's own data directory, the one its |
| +# database is in (discussion #256 item D): a command with no working_dir |
| +# in its manifest starts where the instance starts, and the trampoline |
| +# chdirs to the manifest's working_dir when there is one (rootfs apps |
| +# design §5.2). systemd applies it after the namespace is set up, so the |
| +# bind above is what it chdirs into. Same value as the instance template: |
| +# a command sees the files the instance made from the same place. |
| +WorkingDirectory=/data/instances/%I |
| +# The image is the app (#214 item 2): no app and no live bind, and nothing |
| +# from the version store. The command's root is the image the run was |
| +# admitted against, through root's own cmd link under the exe layout, not |
| +# the run link the instance and its workers mount (merge ruling M4): an |
| +# instance starting on another image around a swap can never repoint the |
| +# source this unit is about to mount, nor the reverse. The control service |
| +# writes it before it starts the unit. |
| +RootImage=/run/carlos-control/exe/%I/rootfs/cmd/image |
| +# The command's own root-written launch file and resolver copy, beside its |
| +# image link, read-only (C4; probe launch-root-owned, dns). The control |
| +# service writes all three sources before it starts the unit. |
| +BindReadOnlyPaths=/run/carlos-control/exe/%I/rootfs/cmd/launch.json:/opt/carlos/launch.json /run/carlos-control/exe/%I/rootfs/cmd/resolv.conf:/etc/resolv.conf |
| + |
| +# --- the rootfs namespace (restated from carlos-instance@.service for an image root; rootfs apps design §5.3) --- |
| +# Every line of the binary block (carlos-instance@.service) is restated |
| +# here for a root that is the instance's image, not the host: a line that |
| +# still binds is kept with its comment; a line that would be a no-op, or |
| +# would fail on a path the image lacks, is replaced by one that proves the |
| +# same property, and the rootfs probe (instance-probe --rootfs) proves |
| +# each replacement from inside. The five rootfs units carry this block |
| +# byte for byte (TestRootfsTemplatesShareOneNamespace), so what the probe |
| +# proves is what a tenant gets. |
| +# |
| +# New, with no binary line to stand in for: the root itself. The image is |
| +# mounted read-only, no set-id bit in it is honoured and no device node in |
| +# it is usable, and /proc, /sys, /dev and /run are fresh API mounts of |
| +# this namespace, not the image's directories. RootImage= itself sits |
| +# outside the block because it is per instance (a drop-in for the |
| +# generation and worker templates). The probe's root-image check: the root |
| +# is a squashfs on a loop device carrying the probe marker. |
| +RootImageOptions=ro,nosuid,nodev |
| +MountAPIVFS=yes |
| +# Kept: on an image root this makes the root and the image's /etc |
| +# read-only (squashfs already is); /proc, /dev and /run stay |
| +# writable-implicit. The probe's root-read-only check: a create at / and |
| +# at /etc answers EROFS. |
| +ProtectSystem=strict |
| +# Kept: the image's /data mount point (ship adds it) becomes an empty |
| +# read-only tmpfs, and the bind outside this block reveals exactly one |
| +# directory in it (probe data-tree). |
| +TemporaryFileSystem=/data:ro |
| +# Kept: /run is a fresh tmpfs under MountAPIVFS=, so this is created, |
| +# never missing (probe run-tree). |
| +TemporaryFileSystem=/run/carlos:ro |
| +# Dropped: TemporaryFileSystem=/var/cache:ro and /var/lib:ro. The binary |
| +# block hides those HOST trees because other carlos-uid units write them |
| +# (codex R9-2); the host's /var is not in an image root at all, and the |
| +# image's own /var/lib is the app's (Debian's /var/lib/gems). What stands |
| +# in for them is the root being the image, proven by the probe's |
| +# root-image check, and its mounts-allowlisted check: no mount in the |
| +# namespace comes from the host's /var, /opt/carlos/versions, |
| +# /opt/carlos/live or another instance's directory. |
| +# |
| +# Kept: the version store holds every app's build on the box, world- |
| +# readable so a unit can exec its own (#214 item 2). A rootfs instance |
| +# sees none of it, and nothing of its image's own /opt/carlos either (ship |
| +# adds the mount point): an empty /opt/carlos with the platform binary |
| +# that ExecStart= and ExecStopPost= run put back, and, outside this block |
| +# because it is per instance, root's launch file at |
| +# /opt/carlos/launch.json. The probe's opt-own-image check lists exactly |
| +# what this line and the per-unit binds put back. |
| +TemporaryFileSystem=/opt/carlos:ro |
| +BindReadOnlyPaths=/opt/carlos/carlos |
| +# Dropped: InaccessiblePaths=/etc/carlos /run/carlos-control. The binary |
| +# block hides root-written credentials a carlos-GROUP process may read |
| +# (/etc/carlos/relay-masters is root:carlos 0640, codex R10-1) and the |
| +# root control service's socket directory (isolated-exec design §5.2). |
| +# Neither path exists in an image root; systemd fails a unit on a missing |
| +# InaccessiblePaths= target, and "-" would make the line a silent no-op. |
| +# The probe's etc-carlos and control-unreachable checks: both answer |
| +# ENOENT from inside. |
| +# |
| +# Kept: hides the image's /home, /root and /run/user; systemd ignores a |
| +# missing one. A unit-file pin only: the property is the image's, not the |
| +# host's. |
| +ProtectHome=yes |
| +# Replaces PrivateTmp=yes, which would bind host directories under |
| +# /var/tmp/systemd-private-* and need /var/tmp in the image. A private |
| +# tmpfs instead (rootfs apps design §5.3): empty, writable, charged to |
| +# this cgroup, and nothing of the host's /tmp in it. The probe's |
| +# tmp-private check: empty, writable, and the host's fixture |
| +# /tmp/carlos-probe absent. |
| +TemporaryFileSystem=/tmp:mode=1777,nosuid,nodev |
| +# Kept: a private /dev on the image's mount point (ship adds it). Probe |
| +# shm-private and the mount allowlist. |
| +PrivateDevices=yes |
| +# Kept: PrivateDevices= bind-mounts the HOST's /dev/shm into the private |
| +# /dev, and every instance is uid carlos, so a sibling's shared memory |
| +# would be one open away. Each unit gets its own, charged to its own |
| +# cgroup (probe shm-private). |
| +TemporaryFileSystem=/dev/shm:mode=1777,nosuid,nodev |
| +# Kept: POSIX message queues and SysV IPC belong to an IPC namespace, and |
| +# every instance is uid carlos, so a sibling's queue would be one mq_open |
| +# away (#178). PrivateIPC= gives each unit its own namespace, and |
| +# mq_open() goes through that namespace's own internal mount. |
| +# PrivateDevices= carries the host's /dev/mqueue into the private /dev the |
| +# way it carries /dev/shm, and that mount would still list and open the |
| +# host namespace's queues, so it is replaced the same way (probe |
| +# ipc-private covers both). |
| +PrivateIPC=yes |
| +TemporaryFileSystem=/dev/mqueue:ro |
| +# New (Plan 2 deviation 8; Plan 6 adds it to the binary block): |
| +# PrivateDevices= carries the host's /dev/pts and /dev/ptmx into the |
| +# private /dev the same way, and a tenant's terminal is what Plan 6 makes |
| +# of them. Hidden here from the start, so Plan 6 never edits a security |
| +# block it did not review. The probe's pts-private check: /dev/pts lists |
| +# nothing and open(/dev/ptmx) fails. |
| +TemporaryFileSystem=/dev/pts:ro |
| +# Replaces ProtectControlGroups=yes. The binary block mounts /sys/fs/cgroup |
| +# read-only so the edge's delegated subtree, carlos-owned, is not one |
| +# write away for a same-uid tenant (codex R9-1); with a fresh API /sys, |
| +# whether cgroup2 is mounted read-only under it is not something the |
| +# binary probe's check (a write to the edge's delegated cgroup.procs) can |
| +# see from an image. strict (systemd 257) is a cgroup namespace with a |
| +# private read-only /sys/fs/cgroup showing only this unit's own subtree. |
| +# The probe's cgroup-strict check: /proc/self/cgroup is 0::/, the mount |
| +# carries ro, access(W_OK) on its own cgroup.procs answers EROFS, and the |
| +# listing names no carlos.slice or system.slice; a systemd that ignores |
| +# the value leaves the listing populated and fails the line. |
| +ProtectControlGroups=strict |
| +# Kept: these apply to the fresh /proc (probe pid1). |
| +ProtectProc=invisible |
| +ProcSubset=pid |
| +PrivatePIDs=yes |
| +# Kept, unchanged meaning; a unit-file pin. NoNewPrivileges= also voids |
| +# any set-id file in the image. |
| +RestrictNamespaces=yes |
| +NoNewPrivileges=yes |
| +CapabilityBoundingSet= |
| +# Kept, unchanged; a unit-file pin. |
| +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX |
| +# Abstract unix sockets live in the network namespace, which instances |
| +# share with the box because they need outbound TCP, and the line above |
| +# cannot tell them from path sockets (#178). The trampoline scopes them |
| +# with Landlock before it execs the app (instanceexec.ScopeAbstractUnix): |
| +# an instance can connect only to abstract sockets its own processes |
| +# bound. A sibling can still bind a name first; that is a refusal of |
| +# service, never a connection. |
| +# |
| +# Egress (#214 item 3): the internet, and nothing on the box or private. |
| +# Loopback is where the console and other tenants' addr upstreams listen; |
| +# RFC 1918, ULA, link-local and CGNAT are the VPC, other boxes and the |
| +# cloud's internal services. IMDS stays named so the property is |
| +# greppable. A cgroup filter: it drops in both directions for every |
| +# socket the unit opens. The one thing an instance needs from these |
| +# ranges is the box's DNS resolver, so the control service re-opens |
| +# exactly the nameservers the instance sees with IPAddressAllow= on |
| +# carlos-instances.slice (an allow wins, and a slice's lists reach every |
| +# unit below it). The probe checks both halves. |
| +# |
| +# Kept byte for byte (the test pins it to the binary block's). The box's |
| +# own addresses are not in this list (#367): the control service denies |
| +# them on carlos-instances.slice, whose list every unit below it is merged |
| +# with, and the probe's box-addr check finds them closed from inside. |
| +IPAddressDeny=localhost multicast 0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 169.254.0.0/16 169.254.169.254/32 172.16.0.0/12 192.0.0.0/24 192.168.0.0/16 198.18.0.0/15 240.0.0.0/4 64:ff9b::/96 fc00::/7 fd00:ec2::254/128 fe80::/10 fec0::/10 |
| +# Replaced, outside this block because it is per instance: |
| +# BindReadOnlyPaths=-/run/systemd/resolve/resolv.conf:/etc/resolv.conf. |
| +# Its "-" skips a box without resolved, and inside an image that would |
| +# leave the image's own (empty) resolv.conf in place: no DNS. Each rootfs |
| +# unit binds root's copy of exactly the file the control service opened on |
| +# the slice (instancectl resolverFile) over /etc/resolv.conf instead. The |
| +# reason the binary line never binds resolved's stub still holds: an allow |
| +# opens an address both ways and on every port, so a loopback resolver can |
| +# never be opened; a tenant could bind systemd-resolved's 127.0.0.53 |
| +# itself and reach every listener bound to all addresses through it. |
| +# Where resolved runs, instances resolve through its upstreams, and the |
| +# control service opens those. The probe's dns and resolver-ports checks |
| +# read /etc/resolv.conf from inside and compare it with the allow. |
| +# |
| +# Kept, unchanged; unit-file pins. |
| +ProtectKernelTunables=yes |
| +ProtectKernelModules=yes |
| +ProtectKernelLogs=yes |
| +ProtectClock=yes |
| +ProtectHostname=yes |
| +RemoveIPC=yes |
| +LockPersonality=yes |
| +RestrictRealtime=yes |
| +# No RestrictSUIDSGID=: systemd cannot read the mode inside openat2's |
| +# open_how, so it refuses openat2 outright (ENOSYS), and every instancefs |
| +# open in here is an openat2 (#126, first roll; systemd 257). What it |
| +# guarded is covered: with no capabilities an instance can only make a |
| +# set-id file owned by carlos, which gains nothing for another carlos |
| +# process, NoNewPrivileges= voids it in here, and no other uid can reach |
| +# one (the image is nosuid by RootImageOptions=, the tenant trees are |
| +# nosuid, and /tmp is a nosuid tmpfs of this unit's own). |
| +# |
| +# Kept in the block for every template, the probe included (R11, rootfs |
| +# apps design §5.11). A release whose root-staged manifest declares jit |
| +# runs without it through a per-start drop-in that root writes and checks |
| +# the effective property of (C10), never through a unit file. The probe's |
| +# mdwe check: an mmap(PROT_WRITE|PROT_EXEC) answers EPERM inside the |
| +# probe, which never gets the drop-in. |
| +MemoryDenyWriteExecute=yes |
| +SystemCallArchitectures=native |
| + |
| +# --- per-service resources; the tier lives on the host slice (§4.3) --- |
| +TasksMax=256 |
| +# The kernel picks a tenant before it picks anything else in carlos.slice. |
| +OOMScoreAdjust=500 |
| +# One OOM kill takes every process of THIS service (systemd sets |
| +# memory.oom.group on the service cgroup for OOMPolicy=kill). A worker is |
| +# a sibling service with its own family rule (isolated-exec design §4.3; |
| +# rootfs apps design §5.4). |
| +OOMPolicy=kill |
diff --git a/internal/host/units/carlos-instance-rootfs-gen@.service b/internal/host/units/carlos-instance-rootfs-gen@.service| new file mode 100644 |
| index 000000000..fff3496df |
| --- /dev/null |
| +++ b/internal/host/units/carlos-instance-rootfs-gen@.service |
| @@ -0,0 +1,239 @@ |
| +# One generation of a rootfs instance (zero-downtime deploys design §7.6; |
| +# rootfs apps design §5.3): carlos-instance-gen@.service for an app |
| +# shipped as an image. The instance string is "<instance-id>:<generation>" |
| +# (%I), and the trampoline and the exit writer split it. systemd has no |
| +# specifier for part of an instance string, and ':' is BindPaths='s field |
| +# separator, so everything keyed by the host ID alone (the slice, the data |
| +# mounts) or by the generation (the image, root's launch file and resolver |
| +# copy, the log socket) comes from a runtime drop-in the control service |
| +# writes before each start. Without that drop-in this unit has no |
| +# RootImage=, an empty /data and no launch file, so its trampoline |
| +# refuses, and the control service will not start it unless systemd |
| +# reports the drop-in's slice. carlos-instance-rootfs@.service is the |
| +# same unit keyed by the host alone. |
| +[Unit] |
| +Description=carlos rootfs instance generation %I |
| +# Stop-only coupling, as carlos-instance@.service (codex R1-7). |
| +BindsTo=carlos-edge.service |
| +Requires=data-instances.mount run-carlos-instances.mount run-carlos-logs.mount |
| +After=carlos-edge.service data-instances.mount run-carlos-instances.mount run-carlos-logs.mount |
| + |
| +[Service] |
| +Type=simple |
| +# --rootfs before the role, in ExecStart= and ExecStopPost= alike, as in |
| +# carlos-instance-rootfs@.service. |
| +ExecStart=/opt/carlos/carlos instance-exec --rootfs run %I |
| +# The tenant runs in its own data directory (#256 D). %I here is |
| +# <id>:<n>, not the host's id, so the real directory comes from the |
| +# generation's runtime drop-in, which overrides this line. This one only |
| +# applies if the drop-in did not, and then the start is already refused |
| +# (the control service checks the drop-in's slice first), and /data is an |
| +# empty tmpfs, so the chdir fails closed. |
| +WorkingDirectory=/data/instances |
| +ExecStopPost=/opt/carlos/carlos instance-exit --rootfs run %I |
| +User=carlos |
| +Group=carlos |
| +# The activator owns recycling; systemd must not race it. |
| +Restart=no |
| +KillMode=mixed |
| +TimeoutStopSec=20 |
| +# The main process is the trampoline, which stays as the namespace's init |
| +# and runs the app as its child (#126): it forwards the SIGTERM and exits |
| +# 128+n when the app dies by signal n. These are SIGHUP, SIGINT, SIGPIPE |
| +# and SIGTERM, the deaths systemd counts as clean for a main process, so a |
| +# stop still leaves the unit inactive rather than failed. |
| +SuccessExitStatus=129 130 141 143 |
| +StandardError=inherit |
| + |
| +# --- the rootfs namespace (restated from carlos-instance@.service for an image root; rootfs apps design §5.3) --- |
| +# Every line of the binary block (carlos-instance@.service) is restated |
| +# here for a root that is the instance's image, not the host: a line that |
| +# still binds is kept with its comment; a line that would be a no-op, or |
| +# would fail on a path the image lacks, is replaced by one that proves the |
| +# same property, and the rootfs probe (instance-probe --rootfs) proves |
| +# each replacement from inside. The five rootfs units carry this block |
| +# byte for byte (TestRootfsTemplatesShareOneNamespace), so what the probe |
| +# proves is what a tenant gets. |
| +# |
| +# New, with no binary line to stand in for: the root itself. The image is |
| +# mounted read-only, no set-id bit in it is honoured and no device node in |
| +# it is usable, and /proc, /sys, /dev and /run are fresh API mounts of |
| +# this namespace, not the image's directories. RootImage= itself sits |
| +# outside the block because it is per instance (a drop-in for the |
| +# generation and worker templates). The probe's root-image check: the root |
| +# is a squashfs on a loop device carrying the probe marker. |
| +RootImageOptions=ro,nosuid,nodev |
| +MountAPIVFS=yes |
| +# Kept: on an image root this makes the root and the image's /etc |
| +# read-only (squashfs already is); /proc, /dev and /run stay |
| +# writable-implicit. The probe's root-read-only check: a create at / and |
| +# at /etc answers EROFS. |
| +ProtectSystem=strict |
| +# Kept: the image's /data mount point (ship adds it) becomes an empty |
| +# read-only tmpfs, and the bind outside this block reveals exactly one |
| +# directory in it (probe data-tree). |
| +TemporaryFileSystem=/data:ro |
| +# Kept: /run is a fresh tmpfs under MountAPIVFS=, so this is created, |
| +# never missing (probe run-tree). |
| +TemporaryFileSystem=/run/carlos:ro |
| +# Dropped: TemporaryFileSystem=/var/cache:ro and /var/lib:ro. The binary |
| +# block hides those HOST trees because other carlos-uid units write them |
| +# (codex R9-2); the host's /var is not in an image root at all, and the |
| +# image's own /var/lib is the app's (Debian's /var/lib/gems). What stands |
| +# in for them is the root being the image, proven by the probe's |
| +# root-image check, and its mounts-allowlisted check: no mount in the |
| +# namespace comes from the host's /var, /opt/carlos/versions, |
| +# /opt/carlos/live or another instance's directory. |
| +# |
| +# Kept: the version store holds every app's build on the box, world- |
| +# readable so a unit can exec its own (#214 item 2). A rootfs instance |
| +# sees none of it, and nothing of its image's own /opt/carlos either (ship |
| +# adds the mount point): an empty /opt/carlos with the platform binary |
| +# that ExecStart= and ExecStopPost= run put back, and, outside this block |
| +# because it is per instance, root's launch file at |
| +# /opt/carlos/launch.json. The probe's opt-own-image check lists exactly |
| +# what this line and the per-unit binds put back. |
| +TemporaryFileSystem=/opt/carlos:ro |
| +BindReadOnlyPaths=/opt/carlos/carlos |
| +# Dropped: InaccessiblePaths=/etc/carlos /run/carlos-control. The binary |
| +# block hides root-written credentials a carlos-GROUP process may read |
| +# (/etc/carlos/relay-masters is root:carlos 0640, codex R10-1) and the |
| +# root control service's socket directory (isolated-exec design §5.2). |
| +# Neither path exists in an image root; systemd fails a unit on a missing |
| +# InaccessiblePaths= target, and "-" would make the line a silent no-op. |
| +# The probe's etc-carlos and control-unreachable checks: both answer |
| +# ENOENT from inside. |
| +# |
| +# Kept: hides the image's /home, /root and /run/user; systemd ignores a |
| +# missing one. A unit-file pin only: the property is the image's, not the |
| +# host's. |
| +ProtectHome=yes |
| +# Replaces PrivateTmp=yes, which would bind host directories under |
| +# /var/tmp/systemd-private-* and need /var/tmp in the image. A private |
| +# tmpfs instead (rootfs apps design §5.3): empty, writable, charged to |
| +# this cgroup, and nothing of the host's /tmp in it. The probe's |
| +# tmp-private check: empty, writable, and the host's fixture |
| +# /tmp/carlos-probe absent. |
| +TemporaryFileSystem=/tmp:mode=1777,nosuid,nodev |
| +# Kept: a private /dev on the image's mount point (ship adds it). Probe |
| +# shm-private and the mount allowlist. |
| +PrivateDevices=yes |
| +# Kept: PrivateDevices= bind-mounts the HOST's /dev/shm into the private |
| +# /dev, and every instance is uid carlos, so a sibling's shared memory |
| +# would be one open away. Each unit gets its own, charged to its own |
| +# cgroup (probe shm-private). |
| +TemporaryFileSystem=/dev/shm:mode=1777,nosuid,nodev |
| +# Kept: POSIX message queues and SysV IPC belong to an IPC namespace, and |
| +# every instance is uid carlos, so a sibling's queue would be one mq_open |
| +# away (#178). PrivateIPC= gives each unit its own namespace, and |
| +# mq_open() goes through that namespace's own internal mount. |
| +# PrivateDevices= carries the host's /dev/mqueue into the private /dev the |
| +# way it carries /dev/shm, and that mount would still list and open the |
| +# host namespace's queues, so it is replaced the same way (probe |
| +# ipc-private covers both). |
| +PrivateIPC=yes |
| +TemporaryFileSystem=/dev/mqueue:ro |
| +# New (Plan 2 deviation 8; Plan 6 adds it to the binary block): |
| +# PrivateDevices= carries the host's /dev/pts and /dev/ptmx into the |
| +# private /dev the same way, and a tenant's terminal is what Plan 6 makes |
| +# of them. Hidden here from the start, so Plan 6 never edits a security |
| +# block it did not review. The probe's pts-private check: /dev/pts lists |
| +# nothing and open(/dev/ptmx) fails. |
| +TemporaryFileSystem=/dev/pts:ro |
| +# Replaces ProtectControlGroups=yes. The binary block mounts /sys/fs/cgroup |
| +# read-only so the edge's delegated subtree, carlos-owned, is not one |
| +# write away for a same-uid tenant (codex R9-1); with a fresh API /sys, |
| +# whether cgroup2 is mounted read-only under it is not something the |
| +# binary probe's check (a write to the edge's delegated cgroup.procs) can |
| +# see from an image. strict (systemd 257) is a cgroup namespace with a |
| +# private read-only /sys/fs/cgroup showing only this unit's own subtree. |
| +# The probe's cgroup-strict check: /proc/self/cgroup is 0::/, the mount |
| +# carries ro, access(W_OK) on its own cgroup.procs answers EROFS, and the |
| +# listing names no carlos.slice or system.slice; a systemd that ignores |
| +# the value leaves the listing populated and fails the line. |
| +ProtectControlGroups=strict |
| +# Kept: these apply to the fresh /proc (probe pid1). |
| +ProtectProc=invisible |
| +ProcSubset=pid |
| +PrivatePIDs=yes |
| +# Kept, unchanged meaning; a unit-file pin. NoNewPrivileges= also voids |
| +# any set-id file in the image. |
| +RestrictNamespaces=yes |
| +NoNewPrivileges=yes |
| +CapabilityBoundingSet= |
| +# Kept, unchanged; a unit-file pin. |
| +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX |
| +# Abstract unix sockets live in the network namespace, which instances |
| +# share with the box because they need outbound TCP, and the line above |
| +# cannot tell them from path sockets (#178). The trampoline scopes them |
| +# with Landlock before it execs the app (instanceexec.ScopeAbstractUnix): |
| +# an instance can connect only to abstract sockets its own processes |
| +# bound. A sibling can still bind a name first; that is a refusal of |
| +# service, never a connection. |
| +# |
| +# Egress (#214 item 3): the internet, and nothing on the box or private. |
| +# Loopback is where the console and other tenants' addr upstreams listen; |
| +# RFC 1918, ULA, link-local and CGNAT are the VPC, other boxes and the |
| +# cloud's internal services. IMDS stays named so the property is |
| +# greppable. A cgroup filter: it drops in both directions for every |
| +# socket the unit opens. The one thing an instance needs from these |
| +# ranges is the box's DNS resolver, so the control service re-opens |
| +# exactly the nameservers the instance sees with IPAddressAllow= on |
| +# carlos-instances.slice (an allow wins, and a slice's lists reach every |
| +# unit below it). The probe checks both halves. |
| +# |
| +# Kept byte for byte (the test pins it to the binary block's). The box's |
| +# own addresses are not in this list (#367): the control service denies |
| +# them on carlos-instances.slice, whose list every unit below it is merged |
| +# with, and the probe's box-addr check finds them closed from inside. |
| +IPAddressDeny=localhost multicast 0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 169.254.0.0/16 169.254.169.254/32 172.16.0.0/12 192.0.0.0/24 192.168.0.0/16 198.18.0.0/15 240.0.0.0/4 64:ff9b::/96 fc00::/7 fd00:ec2::254/128 fe80::/10 fec0::/10 |
| +# Replaced, outside this block because it is per instance: |
| +# BindReadOnlyPaths=-/run/systemd/resolve/resolv.conf:/etc/resolv.conf. |
| +# Its "-" skips a box without resolved, and inside an image that would |
| +# leave the image's own (empty) resolv.conf in place: no DNS. Each rootfs |
| +# unit binds root's copy of exactly the file the control service opened on |
| +# the slice (instancectl resolverFile) over /etc/resolv.conf instead. The |
| +# reason the binary line never binds resolved's stub still holds: an allow |
| +# opens an address both ways and on every port, so a loopback resolver can |
| +# never be opened; a tenant could bind systemd-resolved's 127.0.0.53 |
| +# itself and reach every listener bound to all addresses through it. |
| +# Where resolved runs, instances resolve through its upstreams, and the |
| +# control service opens those. The probe's dns and resolver-ports checks |
| +# read /etc/resolv.conf from inside and compare it with the allow. |
| +# |
| +# Kept, unchanged; unit-file pins. |
| +ProtectKernelTunables=yes |
| +ProtectKernelModules=yes |
| +ProtectKernelLogs=yes |
| +ProtectClock=yes |
| +ProtectHostname=yes |
| +RemoveIPC=yes |
| +LockPersonality=yes |
| +RestrictRealtime=yes |
| +# No RestrictSUIDSGID=: systemd cannot read the mode inside openat2's |
| +# open_how, so it refuses openat2 outright (ENOSYS), and every instancefs |
| +# open in here is an openat2 (#126, first roll; systemd 257). What it |
| +# guarded is covered: with no capabilities an instance can only make a |
| +# set-id file owned by carlos, which gains nothing for another carlos |
| +# process, NoNewPrivileges= voids it in here, and no other uid can reach |
| +# one (the image is nosuid by RootImageOptions=, the tenant trees are |
| +# nosuid, and /tmp is a nosuid tmpfs of this unit's own). |
| +# |
| +# Kept in the block for every template, the probe included (R11, rootfs |
| +# apps design §5.11). A release whose root-staged manifest declares jit |
| +# runs without it through a per-start drop-in that root writes and checks |
| +# the effective property of (C10), never through a unit file. The probe's |
| +# mdwe check: an mmap(PROT_WRITE|PROT_EXEC) answers EPERM inside the |
| +# probe, which never gets the drop-in. |
| +MemoryDenyWriteExecute=yes |
| +SystemCallArchitectures=native |
| + |
| +# --- per-service resources; the tier lives on the host slice (§4.3) --- |
| +TasksMax=256 |
| +# The kernel picks a tenant before it picks anything else in carlos.slice. |
| +OOMScoreAdjust=500 |
| +# One OOM kill takes every process of THIS service (systemd sets |
| +# memory.oom.group on the service cgroup for OOMPolicy=kill). A worker is |
| +# a sibling service with its own family rule (isolated-exec design §4.3; |
| +# rootfs apps design §5.4). |
| +OOMPolicy=kill |
diff --git a/internal/host/units/carlos-instance-rootfs-probe.service b/internal/host/units/carlos-instance-rootfs-probe.service| new file mode 100644 |
| index 000000000..fe4eb2f09 |
| --- /dev/null |
| +++ b/internal/host/units/carlos-instance-rootfs-probe.service |
| @@ -0,0 +1,231 @@ |
| +# The rootfs readiness probe (rootfs apps design §5.3, §8; isolated-exec |
| +# design §8.5): a oneshot under the SAME restated namespace text as the |
| +# rootfs templates (the test pins the block byte for byte), run by |
| +# carlos-instance-control at its start and every five minutes, with a |
| +# probe image for its root so that it sees the image root a rootfs tenant |
| +# gets. From inside it asserts what such a tenant would see and prints a |
| +# verdict; the unit's StandardOutput=truncate: lands it where the control |
| +# service reads it, and a box reports rootfs readiness only after this |
| +# probe passes. It exits 0 whatever the verdict: the verdict is the |
| +# output. Its instance directory is _probe, a name no host can have; the |
| +# tmpfiles lines create it. The binary probe |
| +# (carlos-instance-probe.service) stays as it is. |
| +[Unit] |
| +Description=carlos rootfs isolation probe |
| +Requires=data-instances.mount run-carlos-instances.mount |
| +After=data-instances.mount run-carlos-instances.mount |
| + |
| +[Service] |
| +Type=oneshot |
| +ExecStart=/opt/carlos/carlos instance-probe --rootfs |
| +User=carlos |
| +Group=carlos |
| +Slice=carlos-instances.slice |
| +StandardOutput=truncate:/run/carlos-control/rootfs-probe.out |
| +StandardError=journal |
| +TimeoutStartSec=30 |
| +# The instance's own two directories, the only writable data paths. |
| +BindPaths=/data/instances/_probe /run/carlos/instances/_probe |
| +ReadWritePaths=/data/instances/_probe /run/carlos/instances/_probe |
| +# The probe image, through root's link under the exe layout for the name |
| +# _probe, exactly as a tenant's image is reached (#214 item 2; the image |
| +# is the app). The control service writes it before every probe run. |
| +RootImage=/run/carlos-control/exe/_probe/rootfs/run/image |
| +# The same two root-written files a tenant gets, for the name _probe, and |
| +# one more: the box's own addresses, which the probe must find closed |
| +# (box-addr; the deny lives on carlos-instances.slice, #367). The control |
| +# service writes all three sources before every probe run. |
| +BindReadOnlyPaths=/run/carlos-control/exe/_probe/rootfs/run/launch.json:/opt/carlos/launch.json /run/carlos-control/exe/_probe/rootfs/run/resolv.conf:/etc/resolv.conf /run/carlos-control/exe/_probe/rootfs/run/box-addrs:/opt/carlos/box-addrs |
| + |
| +# --- the rootfs namespace (restated from carlos-instance@.service for an image root; rootfs apps design §5.3) --- |
| +# Every line of the binary block (carlos-instance@.service) is restated |
| +# here for a root that is the instance's image, not the host: a line that |
| +# still binds is kept with its comment; a line that would be a no-op, or |
| +# would fail on a path the image lacks, is replaced by one that proves the |
| +# same property, and the rootfs probe (instance-probe --rootfs) proves |
| +# each replacement from inside. The five rootfs units carry this block |
| +# byte for byte (TestRootfsTemplatesShareOneNamespace), so what the probe |
| +# proves is what a tenant gets. |
| +# |
| +# New, with no binary line to stand in for: the root itself. The image is |
| +# mounted read-only, no set-id bit in it is honoured and no device node in |
| +# it is usable, and /proc, /sys, /dev and /run are fresh API mounts of |
| +# this namespace, not the image's directories. RootImage= itself sits |
| +# outside the block because it is per instance (a drop-in for the |
| +# generation and worker templates). The probe's root-image check: the root |
| +# is a squashfs on a loop device carrying the probe marker. |
| +RootImageOptions=ro,nosuid,nodev |
| +MountAPIVFS=yes |
| +# Kept: on an image root this makes the root and the image's /etc |
| +# read-only (squashfs already is); /proc, /dev and /run stay |
| +# writable-implicit. The probe's root-read-only check: a create at / and |
| +# at /etc answers EROFS. |
| +ProtectSystem=strict |
| +# Kept: the image's /data mount point (ship adds it) becomes an empty |
| +# read-only tmpfs, and the bind outside this block reveals exactly one |
| +# directory in it (probe data-tree). |
| +TemporaryFileSystem=/data:ro |
| +# Kept: /run is a fresh tmpfs under MountAPIVFS=, so this is created, |
| +# never missing (probe run-tree). |
| +TemporaryFileSystem=/run/carlos:ro |
| +# Dropped: TemporaryFileSystem=/var/cache:ro and /var/lib:ro. The binary |
| +# block hides those HOST trees because other carlos-uid units write them |
| +# (codex R9-2); the host's /var is not in an image root at all, and the |
| +# image's own /var/lib is the app's (Debian's /var/lib/gems). What stands |
| +# in for them is the root being the image, proven by the probe's |
| +# root-image check, and its mounts-allowlisted check: no mount in the |
| +# namespace comes from the host's /var, /opt/carlos/versions, |
| +# /opt/carlos/live or another instance's directory. |
| +# |
| +# Kept: the version store holds every app's build on the box, world- |
| +# readable so a unit can exec its own (#214 item 2). A rootfs instance |
| +# sees none of it, and nothing of its image's own /opt/carlos either (ship |
| +# adds the mount point): an empty /opt/carlos with the platform binary |
| +# that ExecStart= and ExecStopPost= run put back, and, outside this block |
| +# because it is per instance, root's launch file at |
| +# /opt/carlos/launch.json. The probe's opt-own-image check lists exactly |
| +# what this line and the per-unit binds put back. |
| +TemporaryFileSystem=/opt/carlos:ro |
| +BindReadOnlyPaths=/opt/carlos/carlos |
| +# Dropped: InaccessiblePaths=/etc/carlos /run/carlos-control. The binary |
| +# block hides root-written credentials a carlos-GROUP process may read |
| +# (/etc/carlos/relay-masters is root:carlos 0640, codex R10-1) and the |
| +# root control service's socket directory (isolated-exec design §5.2). |
| +# Neither path exists in an image root; systemd fails a unit on a missing |
| +# InaccessiblePaths= target, and "-" would make the line a silent no-op. |
| +# The probe's etc-carlos and control-unreachable checks: both answer |
| +# ENOENT from inside. |
| +# |
| +# Kept: hides the image's /home, /root and /run/user; systemd ignores a |
| +# missing one. A unit-file pin only: the property is the image's, not the |
| +# host's. |
| +ProtectHome=yes |
| +# Replaces PrivateTmp=yes, which would bind host directories under |
| +# /var/tmp/systemd-private-* and need /var/tmp in the image. A private |
| +# tmpfs instead (rootfs apps design §5.3): empty, writable, charged to |
| +# this cgroup, and nothing of the host's /tmp in it. The probe's |
| +# tmp-private check: empty, writable, and the host's fixture |
| +# /tmp/carlos-probe absent. |
| +TemporaryFileSystem=/tmp:mode=1777,nosuid,nodev |
| +# Kept: a private /dev on the image's mount point (ship adds it). Probe |
| +# shm-private and the mount allowlist. |
| +PrivateDevices=yes |
| +# Kept: PrivateDevices= bind-mounts the HOST's /dev/shm into the private |
| +# /dev, and every instance is uid carlos, so a sibling's shared memory |
| +# would be one open away. Each unit gets its own, charged to its own |
| +# cgroup (probe shm-private). |
| +TemporaryFileSystem=/dev/shm:mode=1777,nosuid,nodev |
| +# Kept: POSIX message queues and SysV IPC belong to an IPC namespace, and |
| +# every instance is uid carlos, so a sibling's queue would be one mq_open |
| +# away (#178). PrivateIPC= gives each unit its own namespace, and |
| +# mq_open() goes through that namespace's own internal mount. |
| +# PrivateDevices= carries the host's /dev/mqueue into the private /dev the |
| +# way it carries /dev/shm, and that mount would still list and open the |
| +# host namespace's queues, so it is replaced the same way (probe |
| +# ipc-private covers both). |
| +PrivateIPC=yes |
| +TemporaryFileSystem=/dev/mqueue:ro |
| +# New (Plan 2 deviation 8; Plan 6 adds it to the binary block): |
| +# PrivateDevices= carries the host's /dev/pts and /dev/ptmx into the |
| +# private /dev the same way, and a tenant's terminal is what Plan 6 makes |
| +# of them. Hidden here from the start, so Plan 6 never edits a security |
| +# block it did not review. The probe's pts-private check: /dev/pts lists |
| +# nothing and open(/dev/ptmx) fails. |
| +TemporaryFileSystem=/dev/pts:ro |
| +# Replaces ProtectControlGroups=yes. The binary block mounts /sys/fs/cgroup |
| +# read-only so the edge's delegated subtree, carlos-owned, is not one |
| +# write away for a same-uid tenant (codex R9-1); with a fresh API /sys, |
| +# whether cgroup2 is mounted read-only under it is not something the |
| +# binary probe's check (a write to the edge's delegated cgroup.procs) can |
| +# see from an image. strict (systemd 257) is a cgroup namespace with a |
| +# private read-only /sys/fs/cgroup showing only this unit's own subtree. |
| +# The probe's cgroup-strict check: /proc/self/cgroup is 0::/, the mount |
| +# carries ro, access(W_OK) on its own cgroup.procs answers EROFS, and the |
| +# listing names no carlos.slice or system.slice; a systemd that ignores |
| +# the value leaves the listing populated and fails the line. |
| +ProtectControlGroups=strict |
| +# Kept: these apply to the fresh /proc (probe pid1). |
| +ProtectProc=invisible |
| +ProcSubset=pid |
| +PrivatePIDs=yes |
| +# Kept, unchanged meaning; a unit-file pin. NoNewPrivileges= also voids |
| +# any set-id file in the image. |
| +RestrictNamespaces=yes |
| +NoNewPrivileges=yes |
| +CapabilityBoundingSet= |
| +# Kept, unchanged; a unit-file pin. |
| +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX |
| +# Abstract unix sockets live in the network namespace, which instances |
| +# share with the box because they need outbound TCP, and the line above |
| +# cannot tell them from path sockets (#178). The trampoline scopes them |
| +# with Landlock before it execs the app (instanceexec.ScopeAbstractUnix): |
| +# an instance can connect only to abstract sockets its own processes |
| +# bound. A sibling can still bind a name first; that is a refusal of |
| +# service, never a connection. |
| +# |
| +# Egress (#214 item 3): the internet, and nothing on the box or private. |
| +# Loopback is where the console and other tenants' addr upstreams listen; |
| +# RFC 1918, ULA, link-local and CGNAT are the VPC, other boxes and the |
| +# cloud's internal services. IMDS stays named so the property is |
| +# greppable. A cgroup filter: it drops in both directions for every |
| +# socket the unit opens. The one thing an instance needs from these |
| +# ranges is the box's DNS resolver, so the control service re-opens |
| +# exactly the nameservers the instance sees with IPAddressAllow= on |
| +# carlos-instances.slice (an allow wins, and a slice's lists reach every |
| +# unit below it). The probe checks both halves. |
| +# |
| +# Kept byte for byte (the test pins it to the binary block's). The box's |
| +# own addresses are not in this list (#367): the control service denies |
| +# them on carlos-instances.slice, whose list every unit below it is merged |
| +# with, and the probe's box-addr check finds them closed from inside. |
| +IPAddressDeny=localhost multicast 0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 169.254.0.0/16 169.254.169.254/32 172.16.0.0/12 192.0.0.0/24 192.168.0.0/16 198.18.0.0/15 240.0.0.0/4 64:ff9b::/96 fc00::/7 fd00:ec2::254/128 fe80::/10 fec0::/10 |
| +# Replaced, outside this block because it is per instance: |
| +# BindReadOnlyPaths=-/run/systemd/resolve/resolv.conf:/etc/resolv.conf. |
| +# Its "-" skips a box without resolved, and inside an image that would |
| +# leave the image's own (empty) resolv.conf in place: no DNS. Each rootfs |
| +# unit binds root's copy of exactly the file the control service opened on |
| +# the slice (instancectl resolverFile) over /etc/resolv.conf instead. The |
| +# reason the binary line never binds resolved's stub still holds: an allow |
| +# opens an address both ways and on every port, so a loopback resolver can |
| +# never be opened; a tenant could bind systemd-resolved's 127.0.0.53 |
| +# itself and reach every listener bound to all addresses through it. |
| +# Where resolved runs, instances resolve through its upstreams, and the |
| +# control service opens those. The probe's dns and resolver-ports checks |
| +# read /etc/resolv.conf from inside and compare it with the allow. |
| +# |
| +# Kept, unchanged; unit-file pins. |
| +ProtectKernelTunables=yes |
| +ProtectKernelModules=yes |
| +ProtectKernelLogs=yes |
| +ProtectClock=yes |
| +ProtectHostname=yes |
| +RemoveIPC=yes |
| +LockPersonality=yes |
| +RestrictRealtime=yes |
| +# No RestrictSUIDSGID=: systemd cannot read the mode inside openat2's |
| +# open_how, so it refuses openat2 outright (ENOSYS), and every instancefs |
| +# open in here is an openat2 (#126, first roll; systemd 257). What it |
| +# guarded is covered: with no capabilities an instance can only make a |
| +# set-id file owned by carlos, which gains nothing for another carlos |
| +# process, NoNewPrivileges= voids it in here, and no other uid can reach |
| +# one (the image is nosuid by RootImageOptions=, the tenant trees are |
| +# nosuid, and /tmp is a nosuid tmpfs of this unit's own). |
| +# |
| +# Kept in the block for every template, the probe included (R11, rootfs |
| +# apps design §5.11). A release whose root-staged manifest declares jit |
| +# runs without it through a per-start drop-in that root writes and checks |
| +# the effective property of (C10), never through a unit file. The probe's |
| +# mdwe check: an mmap(PROT_WRITE|PROT_EXEC) answers EPERM inside the |
| +# probe, which never gets the drop-in. |
| +MemoryDenyWriteExecute=yes |
| +SystemCallArchitectures=native |
| + |
| +# --- per-service resources; the tier lives on the host slice (§4.3) --- |
| +TasksMax=64 |
| +MemoryMax=64M |
| +# Read back by the probe from its own cgroup (#214 item 1): proof that CPU |
| +# and IO settings take under carlos-instances.slice, where the control |
| +# service sets each host slice's from its tier. |
| +CPUQuota=50% |
| +CPUWeight=50 |
| +IOWeight=50 |
diff --git a/internal/host/units/carlos-instance-rootfs-worker@.service b/internal/host/units/carlos-instance-rootfs-worker@.service| new file mode 100644 |
| index 000000000..14094b19f |
| --- /dev/null |
| +++ b/internal/host/units/carlos-instance-rootfs-worker@.service |
| @@ -0,0 +1,251 @@ |
| +# One declared worker process of a rootfs release (rootfs apps design |
| +# §5.4): a sibling of the instance in the same slice, running from the |
| +# same image with the same uid, namespace and reach, the way a sidecar |
| +# does for a binary instance. The instance string is |
| +# "<instance-id>:<worker>" (%I), and the trampoline and the exit writer |
| +# split it. As for a generation, systemd has no specifier for part of an |
| +# instance string and ':' is BindPaths='s field separator, so everything |
| +# keyed by the host ID alone (the slice, the data mounts) or by the worker |
| +# (the image, root's launch file and resolver copy, the log socket) comes |
| +# from a runtime drop-in the control service writes before each start. |
| +# Without that drop-in this unit has no RootImage=, an empty /data and no |
| +# launch file, so its trampoline refuses, and the control service will not |
| +# start it unless systemd reports the drop-in's slice. |
| +[Unit] |
| +Description=carlos rootfs worker %I |
| +# Stop-only coupling: when the edge stops (including the stop half of a |
| +# restart) a worker stops, and nothing starts it again but the agent. |
| +# PartOf= would propagate the RESTART and race the agent's own startup |
| +# (codex R1-7). |
| +BindsTo=carlos-edge.service |
| +Requires=data-instances.mount run-carlos-instances.mount run-carlos-logs.mount |
| +After=carlos-edge.service data-instances.mount run-carlos-instances.mount run-carlos-logs.mount |
| + |
| +[Service] |
| +Type=simple |
| +# --rootfs before the role, in ExecStart= and ExecStopPost= alike: worker |
| +# is a role the trampoline and instance-exit take only with it (there is |
| +# no binary worker unit). |
| +ExecStart=/opt/carlos/carlos instance-exec --rootfs worker %I |
| +# The worker runs in the instance's own data directory, the one the |
| +# instance's database is in (#256 D). %I here is <id>:<worker>, not the |
| +# host's id, so the real directory comes from the worker's runtime |
| +# drop-in, which overrides this line. This one only applies if the |
| +# drop-in did not, and then the start is already refused (the control |
| +# service checks the drop-in's slice first), and /data is an empty tmpfs, |
| +# so the chdir fails closed. |
| +WorkingDirectory=/data/instances |
| +# Runs inside the same namespace, as carlos, after this invocation's |
| +# processes have been signalled: leaves an ADVISORY exit record in the |
| +# instance's own run directory (isolated-exec design §6.5). Not "+" |
| +# (codex R3-2). |
| +ExecStopPost=/opt/carlos/carlos instance-exit --rootfs worker %I |
| +User=carlos |
| +Group=carlos |
| +# The activator owns recycling: a crash stands the worker down and the |
| +# instance's next wake starts it again. systemd must not race that. |
| +Restart=no |
| +# SIGTERM to the worker, SIGKILL to anything it left behind after |
| +# TimeoutStopSec; the activator's stop budget is 20s. |
| +KillMode=mixed |
| +TimeoutStopSec=20 |
| +# The main process is the trampoline, which stays as the namespace's init |
| +# and runs the worker as its child (#126): it forwards the SIGTERM and |
| +# exits 128+n when the worker dies by signal n. These are SIGHUP, SIGINT, |
| +# SIGPIPE and SIGTERM, the deaths systemd counts as clean for a main |
| +# process, so a stop still leaves the unit inactive rather than failed. |
| +SuccessExitStatus=129 130 141 143 |
| +StandardError=inherit |
| + |
| +# --- the rootfs namespace (restated from carlos-instance@.service for an image root; rootfs apps design §5.3) --- |
| +# Every line of the binary block (carlos-instance@.service) is restated |
| +# here for a root that is the instance's image, not the host: a line that |
| +# still binds is kept with its comment; a line that would be a no-op, or |
| +# would fail on a path the image lacks, is replaced by one that proves the |
| +# same property, and the rootfs probe (instance-probe --rootfs) proves |
| +# each replacement from inside. The five rootfs units carry this block |
| +# byte for byte (TestRootfsTemplatesShareOneNamespace), so what the probe |
| +# proves is what a tenant gets. |
| +# |
| +# New, with no binary line to stand in for: the root itself. The image is |
| +# mounted read-only, no set-id bit in it is honoured and no device node in |
| +# it is usable, and /proc, /sys, /dev and /run are fresh API mounts of |
| +# this namespace, not the image's directories. RootImage= itself sits |
| +# outside the block because it is per instance (a drop-in for the |
| +# generation and worker templates). The probe's root-image check: the root |
| +# is a squashfs on a loop device carrying the probe marker. |
| +RootImageOptions=ro,nosuid,nodev |
| +MountAPIVFS=yes |
| +# Kept: on an image root this makes the root and the image's /etc |
| +# read-only (squashfs already is); /proc, /dev and /run stay |
| +# writable-implicit. The probe's root-read-only check: a create at / and |
| +# at /etc answers EROFS. |
| +ProtectSystem=strict |
| +# Kept: the image's /data mount point (ship adds it) becomes an empty |
| +# read-only tmpfs, and the bind outside this block reveals exactly one |
| +# directory in it (probe data-tree). |
| +TemporaryFileSystem=/data:ro |
| +# Kept: /run is a fresh tmpfs under MountAPIVFS=, so this is created, |
| +# never missing (probe run-tree). |
| +TemporaryFileSystem=/run/carlos:ro |
| +# Dropped: TemporaryFileSystem=/var/cache:ro and /var/lib:ro. The binary |
| +# block hides those HOST trees because other carlos-uid units write them |
| +# (codex R9-2); the host's /var is not in an image root at all, and the |
| +# image's own /var/lib is the app's (Debian's /var/lib/gems). What stands |
| +# in for them is the root being the image, proven by the probe's |
| +# root-image check, and its mounts-allowlisted check: no mount in the |
| +# namespace comes from the host's /var, /opt/carlos/versions, |
| +# /opt/carlos/live or another instance's directory. |
| +# |
| +# Kept: the version store holds every app's build on the box, world- |
| +# readable so a unit can exec its own (#214 item 2). A rootfs instance |
| +# sees none of it, and nothing of its image's own /opt/carlos either (ship |
| +# adds the mount point): an empty /opt/carlos with the platform binary |
| +# that ExecStart= and ExecStopPost= run put back, and, outside this block |
| +# because it is per instance, root's launch file at |
| +# /opt/carlos/launch.json. The probe's opt-own-image check lists exactly |
| +# what this line and the per-unit binds put back. |
| +TemporaryFileSystem=/opt/carlos:ro |
| +BindReadOnlyPaths=/opt/carlos/carlos |
| +# Dropped: InaccessiblePaths=/etc/carlos /run/carlos-control. The binary |
| +# block hides root-written credentials a carlos-GROUP process may read |
| +# (/etc/carlos/relay-masters is root:carlos 0640, codex R10-1) and the |
| +# root control service's socket directory (isolated-exec design §5.2). |
| +# Neither path exists in an image root; systemd fails a unit on a missing |
| +# InaccessiblePaths= target, and "-" would make the line a silent no-op. |
| +# The probe's etc-carlos and control-unreachable checks: both answer |
| +# ENOENT from inside. |
| +# |
| +# Kept: hides the image's /home, /root and /run/user; systemd ignores a |
| +# missing one. A unit-file pin only: the property is the image's, not the |
| +# host's. |
| +ProtectHome=yes |
| +# Replaces PrivateTmp=yes, which would bind host directories under |
| +# /var/tmp/systemd-private-* and need /var/tmp in the image. A private |
| +# tmpfs instead (rootfs apps design §5.3): empty, writable, charged to |
| +# this cgroup, and nothing of the host's /tmp in it. The probe's |
| +# tmp-private check: empty, writable, and the host's fixture |
| +# /tmp/carlos-probe absent. |
| +TemporaryFileSystem=/tmp:mode=1777,nosuid,nodev |
| +# Kept: a private /dev on the image's mount point (ship adds it). Probe |
| +# shm-private and the mount allowlist. |
| +PrivateDevices=yes |
| +# Kept: PrivateDevices= bind-mounts the HOST's /dev/shm into the private |
| +# /dev, and every instance is uid carlos, so a sibling's shared memory |
| +# would be one open away. Each unit gets its own, charged to its own |
| +# cgroup (probe shm-private). |
| +TemporaryFileSystem=/dev/shm:mode=1777,nosuid,nodev |
| +# Kept: POSIX message queues and SysV IPC belong to an IPC namespace, and |
| +# every instance is uid carlos, so a sibling's queue would be one mq_open |
| +# away (#178). PrivateIPC= gives each unit its own namespace, and |
| +# mq_open() goes through that namespace's own internal mount. |
| +# PrivateDevices= carries the host's /dev/mqueue into the private /dev the |
| +# way it carries /dev/shm, and that mount would still list and open the |
| +# host namespace's queues, so it is replaced the same way (probe |
| +# ipc-private covers both). |
| +PrivateIPC=yes |
| +TemporaryFileSystem=/dev/mqueue:ro |
| +# New (Plan 2 deviation 8; Plan 6 adds it to the binary block): |
| +# PrivateDevices= carries the host's /dev/pts and /dev/ptmx into the |
| +# private /dev the same way, and a tenant's terminal is what Plan 6 makes |
| +# of them. Hidden here from the start, so Plan 6 never edits a security |
| +# block it did not review. The probe's pts-private check: /dev/pts lists |
| +# nothing and open(/dev/ptmx) fails. |
| +TemporaryFileSystem=/dev/pts:ro |
| +# Replaces ProtectControlGroups=yes. The binary block mounts /sys/fs/cgroup |
| +# read-only so the edge's delegated subtree, carlos-owned, is not one |
| +# write away for a same-uid tenant (codex R9-1); with a fresh API /sys, |
| +# whether cgroup2 is mounted read-only under it is not something the |
| +# binary probe's check (a write to the edge's delegated cgroup.procs) can |
| +# see from an image. strict (systemd 257) is a cgroup namespace with a |
| +# private read-only /sys/fs/cgroup showing only this unit's own subtree. |
| +# The probe's cgroup-strict check: /proc/self/cgroup is 0::/, the mount |
| +# carries ro, access(W_OK) on its own cgroup.procs answers EROFS, and the |
| +# listing names no carlos.slice or system.slice; a systemd that ignores |
| +# the value leaves the listing populated and fails the line. |
| +ProtectControlGroups=strict |
| +# Kept: these apply to the fresh /proc (probe pid1). |
| +ProtectProc=invisible |
| +ProcSubset=pid |
| +PrivatePIDs=yes |
| +# Kept, unchanged meaning; a unit-file pin. NoNewPrivileges= also voids |
| +# any set-id file in the image. |
| +RestrictNamespaces=yes |
| +NoNewPrivileges=yes |
| +CapabilityBoundingSet= |
| +# Kept, unchanged; a unit-file pin. |
| +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX |
| +# Abstract unix sockets live in the network namespace, which instances |
| +# share with the box because they need outbound TCP, and the line above |
| +# cannot tell them from path sockets (#178). The trampoline scopes them |
| +# with Landlock before it execs the app (instanceexec.ScopeAbstractUnix): |
| +# an instance can connect only to abstract sockets its own processes |
| +# bound. A sibling can still bind a name first; that is a refusal of |
| +# service, never a connection. |
| +# |
| +# Egress (#214 item 3): the internet, and nothing on the box or private. |
| +# Loopback is where the console and other tenants' addr upstreams listen; |
| +# RFC 1918, ULA, link-local and CGNAT are the VPC, other boxes and the |
| +# cloud's internal services. IMDS stays named so the property is |
| +# greppable. A cgroup filter: it drops in both directions for every |
| +# socket the unit opens. The one thing an instance needs from these |
| +# ranges is the box's DNS resolver, so the control service re-opens |
| +# exactly the nameservers the instance sees with IPAddressAllow= on |
| +# carlos-instances.slice (an allow wins, and a slice's lists reach every |
| +# unit below it). The probe checks both halves. |
| +# |
| +# Kept byte for byte (the test pins it to the binary block's). The box's |
| +# own addresses are not in this list (#367): the control service denies |
| +# them on carlos-instances.slice, whose list every unit below it is merged |
| +# with, and the probe's box-addr check finds them closed from inside. |
| +IPAddressDeny=localhost multicast 0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 169.254.0.0/16 169.254.169.254/32 172.16.0.0/12 192.0.0.0/24 192.168.0.0/16 198.18.0.0/15 240.0.0.0/4 64:ff9b::/96 fc00::/7 fd00:ec2::254/128 fe80::/10 fec0::/10 |
| +# Replaced, outside this block because it is per instance: |
| +# BindReadOnlyPaths=-/run/systemd/resolve/resolv.conf:/etc/resolv.conf. |
| +# Its "-" skips a box without resolved, and inside an image that would |
| +# leave the image's own (empty) resolv.conf in place: no DNS. Each rootfs |
| +# unit binds root's copy of exactly the file the control service opened on |
| +# the slice (instancectl resolverFile) over /etc/resolv.conf instead. The |
| +# reason the binary line never binds resolved's stub still holds: an allow |
| +# opens an address both ways and on every port, so a loopback resolver can |
| +# never be opened; a tenant could bind systemd-resolved's 127.0.0.53 |
| +# itself and reach every listener bound to all addresses through it. |
| +# Where resolved runs, instances resolve through its upstreams, and the |
| +# control service opens those. The probe's dns and resolver-ports checks |
| +# read /etc/resolv.conf from inside and compare it with the allow. |
| +# |
| +# Kept, unchanged; unit-file pins. |
| +ProtectKernelTunables=yes |
| +ProtectKernelModules=yes |
| +ProtectKernelLogs=yes |
| +ProtectClock=yes |
| +ProtectHostname=yes |
| +RemoveIPC=yes |
| +LockPersonality=yes |
| +RestrictRealtime=yes |
| +# No RestrictSUIDSGID=: systemd cannot read the mode inside openat2's |
| +# open_how, so it refuses openat2 outright (ENOSYS), and every instancefs |
| +# open in here is an openat2 (#126, first roll; systemd 257). What it |
| +# guarded is covered: with no capabilities an instance can only make a |
| +# set-id file owned by carlos, which gains nothing for another carlos |
| +# process, NoNewPrivileges= voids it in here, and no other uid can reach |
| +# one (the image is nosuid by RootImageOptions=, the tenant trees are |
| +# nosuid, and /tmp is a nosuid tmpfs of this unit's own). |
| +# |
| +# Kept in the block for every template, the probe included (R11, rootfs |
| +# apps design §5.11). A release whose root-staged manifest declares jit |
| +# runs without it through a per-start drop-in that root writes and checks |
| +# the effective property of (C10), never through a unit file. The probe's |
| +# mdwe check: an mmap(PROT_WRITE|PROT_EXEC) answers EPERM inside the |
| +# probe, which never gets the drop-in. |
| +MemoryDenyWriteExecute=yes |
| +SystemCallArchitectures=native |
| + |
| +# --- per-service resources; the tier lives on the host slice (§4.3) --- |
| +TasksMax=256 |
| +# The kernel picks a tenant before it picks anything else in carlos.slice. |
| +OOMScoreAdjust=500 |
| +# One OOM kill takes every process of THIS service (systemd sets |
| +# memory.oom.group on the service cgroup for OOMPolicy=kill). A worker is |
| +# a sibling service with its own family rule (isolated-exec design §4.3; |
| +# rootfs apps design §5.4). |
| +OOMPolicy=kill |
diff --git a/internal/host/units/carlos-instance-rootfs@.service b/internal/host/units/carlos-instance-rootfs@.service| new file mode 100644 |
| index 000000000..45965d43d |
| --- /dev/null |
| +++ b/internal/host/units/carlos-instance-rootfs@.service |
| @@ -0,0 +1,281 @@ |
| +# One rootfs instance (rootfs apps design §5.3): carlos-instance@.service |
| +# for an app shipped as an image rather than a binary. The activator owns |
| +# it exactly as it owns a binary instance: lease, restore, seed, |
| +# replication, hibernation, control socket, logs. The process is a unit, |
| +# still uid carlos, whose root is the instance's own image (RootImage=), |
| +# inside a mount namespace in which nothing of the host exists but the |
| +# instance's own two directories, the platform binary and root's launch |
| +# file, on mounts where no symlink is ever followed. The instance name is |
| +# the escaped bounded identifier (%i); paths use the unescaped one (%I). |
| +[Unit] |
| +Description=carlos rootfs instance %I |
| +# Stop-only coupling: when the edge stops (including the stop half of a |
| +# restart) this unit stops, and nothing starts it again but the agent. |
| +# PartOf= would propagate the RESTART and race the agent's own startup |
| +# (codex R1-7). |
| +BindsTo=carlos-edge.service |
| +Requires=data-instances.mount run-carlos-instances.mount run-carlos-logs.mount |
| +After=carlos-edge.service data-instances.mount run-carlos-instances.mount run-carlos-logs.mount |
| + |
| +[Service] |
| +Type=simple |
| +# --rootfs before the role, in ExecStart= and ExecStopPost= alike: the |
| +# trampoline takes the rootfs launch shape (root's launch file names the |
| +# command, the tenant's names only the environment; rootfs apps design |
| +# §5.2), and instance-exit takes the rootfs roles only with it. |
| +ExecStart=/opt/carlos/carlos instance-exec --rootfs run %I |
| +# Runs inside the same namespace, as carlos, after this invocation's |
| +# processes have been signalled: leaves an ADVISORY exit record in the |
| +# instance's own run directory (isolated-exec design §6.5). Not "+": the |
| +# "+" prefix drops User= and the Private*/Protect* settings but keeps the |
| +# explicit TemporaryFileSystem=/BindPaths= mounts (codex R3-2), so nothing |
| +# run from here can see the host's /run/carlos anyway. |
| +ExecStopPost=/opt/carlos/carlos instance-exit --rootfs run %I |
| +User=carlos |
| +Group=carlos |
| +# One slice per host holds the instance, its workers and its command, and |
| +# carries the tier (isolated-exec design §4.3, §6.6); it is the slice a |
| +# binary instance of the same host would sit in. Slices referenced here |
| +# are created on demand; no per-host unit file exists. |
| +Slice=carlos-instances-%i.slice |
| +# The activator owns recycling: a crash stands the entry down and the next |
| +# request wakes it. systemd must not race that. |
| +Restart=no |
| +# SIGTERM to the app, SIGKILL to anything it left behind after |
| +# TimeoutStopSec; the activator's stop budget is 20s. |
| +KillMode=mixed |
| +TimeoutStopSec=20 |
| +# The main process is the trampoline, which stays as the namespace's init |
| +# and runs the app as its child (#126): it forwards the SIGTERM and exits |
| +# 128+n when the app dies by signal n. These are SIGHUP, SIGINT, SIGPIPE |
| +# and SIGTERM, the deaths systemd counts as clean for a main process, so a |
| +# stop still leaves the unit inactive rather than failed. |
| +SuccessExitStatus=129 130 141 143 |
| +# The log stream: stdout and stderr connect to the agent's per-role |
| +# listener (isolated-exec design §6.4). PID 1 opens this as root, outside |
| +# the namespace, on the nosymfollow logs mount, at a path no tenant mount |
| +# contains. |
| +StandardOutput=file:/run/carlos/logs/%I.run.sock |
| +StandardError=inherit |
| +# The instance's own two directories, the only writable data paths, bound |
| +# into the image root at the mount points ship adds (rootfs apps design |
| +# §5.1). |
| +BindPaths=/data/instances/%I /run/carlos/instances/%I |
| +ReadWritePaths=/data/instances/%I /run/carlos/instances/%I |
| +# The working directory is the instance's own data directory, the one its |
| +# database is in (discussion #256 item D): a process with no working_dir |
| +# in its manifest starts where a binary starts, and the trampoline chdirs |
| +# to the manifest's working_dir when there is one (rootfs apps design |
| +# §5.2). systemd applies it after the namespace is set up, so the bind |
| +# above is what it chdirs into. Same value in the cmd template: a command |
| +# sees the files the instance made from the same place. |
| +WorkingDirectory=/data/instances/%I |
| +# The image is the app (#214 item 2): no app and no live bind, and nothing |
| +# from the version store. The root is the instance's own staged image, |
| +# reached through root's per-start link under the control service's exe |
| +# layout for this unit, which the control service writes after checking |
| +# the image against the registry row and the staged manifest (C2, C3). |
| +# PID 1 chases the link at every start, so one instance sees its own |
| +# image and no other. |
| +RootImage=/run/carlos-control/exe/%I/rootfs/run/image |
| +# Two root-written files, read-only, beside the image link: the launch |
| +# file naming the command root resolved from the staged manifest, the |
| +# only thing the trampoline will exec (C4; probe launch-root-owned), and |
| +# root's copy of exactly the resolver file the control service opened on |
| +# the slice, standing in for the binary block's resolved bind (probe dns, |
| +# resolver-ports). The control service writes all three sources before it |
| +# starts the unit. |
| +BindReadOnlyPaths=/run/carlos-control/exe/%I/rootfs/run/launch.json:/opt/carlos/launch.json /run/carlos-control/exe/%I/rootfs/run/resolv.conf:/etc/resolv.conf |
| + |
| +# --- the rootfs namespace (restated from carlos-instance@.service for an image root; rootfs apps design §5.3) --- |
| +# Every line of the binary block (carlos-instance@.service) is restated |
| +# here for a root that is the instance's image, not the host: a line that |
| +# still binds is kept with its comment; a line that would be a no-op, or |
| +# would fail on a path the image lacks, is replaced by one that proves the |
| +# same property, and the rootfs probe (instance-probe --rootfs) proves |
| +# each replacement from inside. The five rootfs units carry this block |
| +# byte for byte (TestRootfsTemplatesShareOneNamespace), so what the probe |
| +# proves is what a tenant gets. |
| +# |
| +# New, with no binary line to stand in for: the root itself. The image is |
| +# mounted read-only, no set-id bit in it is honoured and no device node in |
| +# it is usable, and /proc, /sys, /dev and /run are fresh API mounts of |
| +# this namespace, not the image's directories. RootImage= itself sits |
| +# outside the block because it is per instance (a drop-in for the |
| +# generation and worker templates). The probe's root-image check: the root |
| +# is a squashfs on a loop device carrying the probe marker. |
| +RootImageOptions=ro,nosuid,nodev |
| +MountAPIVFS=yes |
| +# Kept: on an image root this makes the root and the image's /etc |
| +# read-only (squashfs already is); /proc, /dev and /run stay |
| +# writable-implicit. The probe's root-read-only check: a create at / and |
| +# at /etc answers EROFS. |
| +ProtectSystem=strict |
| +# Kept: the image's /data mount point (ship adds it) becomes an empty |
| +# read-only tmpfs, and the bind outside this block reveals exactly one |
| +# directory in it (probe data-tree). |
| +TemporaryFileSystem=/data:ro |
| +# Kept: /run is a fresh tmpfs under MountAPIVFS=, so this is created, |
| +# never missing (probe run-tree). |
| +TemporaryFileSystem=/run/carlos:ro |
| +# Dropped: TemporaryFileSystem=/var/cache:ro and /var/lib:ro. The binary |
| +# block hides those HOST trees because other carlos-uid units write them |
| +# (codex R9-2); the host's /var is not in an image root at all, and the |
| +# image's own /var/lib is the app's (Debian's /var/lib/gems). What stands |
| +# in for them is the root being the image, proven by the probe's |
| +# root-image check, and its mounts-allowlisted check: no mount in the |
| +# namespace comes from the host's /var, /opt/carlos/versions, |
| +# /opt/carlos/live or another instance's directory. |
| +# |
| +# Kept: the version store holds every app's build on the box, world- |
| +# readable so a unit can exec its own (#214 item 2). A rootfs instance |
| +# sees none of it, and nothing of its image's own /opt/carlos either (ship |
| +# adds the mount point): an empty /opt/carlos with the platform binary |
| +# that ExecStart= and ExecStopPost= run put back, and, outside this block |
| +# because it is per instance, root's launch file at |
| +# /opt/carlos/launch.json. The probe's opt-own-image check lists exactly |
| +# what this line and the per-unit binds put back. |
| +TemporaryFileSystem=/opt/carlos:ro |
| +BindReadOnlyPaths=/opt/carlos/carlos |
| +# Dropped: InaccessiblePaths=/etc/carlos /run/carlos-control. The binary |
| +# block hides root-written credentials a carlos-GROUP process may read |
| +# (/etc/carlos/relay-masters is root:carlos 0640, codex R10-1) and the |
| +# root control service's socket directory (isolated-exec design §5.2). |
| +# Neither path exists in an image root; systemd fails a unit on a missing |
| +# InaccessiblePaths= target, and "-" would make the line a silent no-op. |
| +# The probe's etc-carlos and control-unreachable checks: both answer |
| +# ENOENT from inside. |
| +# |
| +# Kept: hides the image's /home, /root and /run/user; systemd ignores a |
| +# missing one. A unit-file pin only: the property is the image's, not the |
| +# host's. |
| +ProtectHome=yes |
| +# Replaces PrivateTmp=yes, which would bind host directories under |
| +# /var/tmp/systemd-private-* and need /var/tmp in the image. A private |
| +# tmpfs instead (rootfs apps design §5.3): empty, writable, charged to |
| +# this cgroup, and nothing of the host's /tmp in it. The probe's |
| +# tmp-private check: empty, writable, and the host's fixture |
| +# /tmp/carlos-probe absent. |
| +TemporaryFileSystem=/tmp:mode=1777,nosuid,nodev |
| +# Kept: a private /dev on the image's mount point (ship adds it). Probe |
| +# shm-private and the mount allowlist. |
| +PrivateDevices=yes |
| +# Kept: PrivateDevices= bind-mounts the HOST's /dev/shm into the private |
| +# /dev, and every instance is uid carlos, so a sibling's shared memory |
| +# would be one open away. Each unit gets its own, charged to its own |
| +# cgroup (probe shm-private). |
| +TemporaryFileSystem=/dev/shm:mode=1777,nosuid,nodev |
| +# Kept: POSIX message queues and SysV IPC belong to an IPC namespace, and |
| +# every instance is uid carlos, so a sibling's queue would be one mq_open |
| +# away (#178). PrivateIPC= gives each unit its own namespace, and |
| +# mq_open() goes through that namespace's own internal mount. |
| +# PrivateDevices= carries the host's /dev/mqueue into the private /dev the |
| +# way it carries /dev/shm, and that mount would still list and open the |
| +# host namespace's queues, so it is replaced the same way (probe |
| +# ipc-private covers both). |
| +PrivateIPC=yes |
| +TemporaryFileSystem=/dev/mqueue:ro |
| +# New (Plan 2 deviation 8; Plan 6 adds it to the binary block): |
| +# PrivateDevices= carries the host's /dev/pts and /dev/ptmx into the |
| +# private /dev the same way, and a tenant's terminal is what Plan 6 makes |
| +# of them. Hidden here from the start, so Plan 6 never edits a security |
| +# block it did not review. The probe's pts-private check: /dev/pts lists |
| +# nothing and open(/dev/ptmx) fails. |
| +TemporaryFileSystem=/dev/pts:ro |
| +# Replaces ProtectControlGroups=yes. The binary block mounts /sys/fs/cgroup |
| +# read-only so the edge's delegated subtree, carlos-owned, is not one |
| +# write away for a same-uid tenant (codex R9-1); with a fresh API /sys, |
| +# whether cgroup2 is mounted read-only under it is not something the |
| +# binary probe's check (a write to the edge's delegated cgroup.procs) can |
| +# see from an image. strict (systemd 257) is a cgroup namespace with a |
| +# private read-only /sys/fs/cgroup showing only this unit's own subtree. |
| +# The probe's cgroup-strict check: /proc/self/cgroup is 0::/, the mount |
| +# carries ro, access(W_OK) on its own cgroup.procs answers EROFS, and the |
| +# listing names no carlos.slice or system.slice; a systemd that ignores |
| +# the value leaves the listing populated and fails the line. |
| +ProtectControlGroups=strict |
| +# Kept: these apply to the fresh /proc (probe pid1). |
| +ProtectProc=invisible |
| +ProcSubset=pid |
| +PrivatePIDs=yes |
| +# Kept, unchanged meaning; a unit-file pin. NoNewPrivileges= also voids |
| +# any set-id file in the image. |
| +RestrictNamespaces=yes |
| +NoNewPrivileges=yes |
| +CapabilityBoundingSet= |
| +# Kept, unchanged; a unit-file pin. |
| +RestrictAddressFamilies=AF_INET AF_INET6 AF_UNIX |
| +# Abstract unix sockets live in the network namespace, which instances |
| +# share with the box because they need outbound TCP, and the line above |
| +# cannot tell them from path sockets (#178). The trampoline scopes them |
| +# with Landlock before it execs the app (instanceexec.ScopeAbstractUnix): |
| +# an instance can connect only to abstract sockets its own processes |
| +# bound. A sibling can still bind a name first; that is a refusal of |
| +# service, never a connection. |
| +# |
| +# Egress (#214 item 3): the internet, and nothing on the box or private. |
| +# Loopback is where the console and other tenants' addr upstreams listen; |
| +# RFC 1918, ULA, link-local and CGNAT are the VPC, other boxes and the |
| +# cloud's internal services. IMDS stays named so the property is |
| +# greppable. A cgroup filter: it drops in both directions for every |
| +# socket the unit opens. The one thing an instance needs from these |
| +# ranges is the box's DNS resolver, so the control service re-opens |
| +# exactly the nameservers the instance sees with IPAddressAllow= on |
| +# carlos-instances.slice (an allow wins, and a slice's lists reach every |
| +# unit below it). The probe checks both halves. |
| +# |
| +# Kept byte for byte (the test pins it to the binary block's). The box's |
| +# own addresses are not in this list (#367): the control service denies |
| +# them on carlos-instances.slice, whose list every unit below it is merged |
| +# with, and the probe's box-addr check finds them closed from inside. |
| +IPAddressDeny=localhost multicast 0.0.0.0/8 10.0.0.0/8 100.64.0.0/10 169.254.0.0/16 169.254.169.254/32 172.16.0.0/12 192.0.0.0/24 192.168.0.0/16 198.18.0.0/15 240.0.0.0/4 64:ff9b::/96 fc00::/7 fd00:ec2::254/128 fe80::/10 fec0::/10 |
| +# Replaced, outside this block because it is per instance: |
| +# BindReadOnlyPaths=-/run/systemd/resolve/resolv.conf:/etc/resolv.conf. |
| +# Its "-" skips a box without resolved, and inside an image that would |
| +# leave the image's own (empty) resolv.conf in place: no DNS. Each rootfs |
| +# unit binds root's copy of exactly the file the control service opened on |
| +# the slice (instancectl resolverFile) over /etc/resolv.conf instead. The |
| +# reason the binary line never binds resolved's stub still holds: an allow |
| +# opens an address both ways and on every port, so a loopback resolver can |
| +# never be opened; a tenant could bind systemd-resolved's 127.0.0.53 |
| +# itself and reach every listener bound to all addresses through it. |
| +# Where resolved runs, instances resolve through its upstreams, and the |
| +# control service opens those. The probe's dns and resolver-ports checks |
| +# read /etc/resolv.conf from inside and compare it with the allow. |
| +# |
| +# Kept, unchanged; unit-file pins. |
| +ProtectKernelTunables=yes |
| +ProtectKernelModules=yes |
| +ProtectKernelLogs=yes |
| +ProtectClock=yes |
| +ProtectHostname=yes |
| +RemoveIPC=yes |
| +LockPersonality=yes |
| +RestrictRealtime=yes |
| +# No RestrictSUIDSGID=: systemd cannot read the mode inside openat2's |
| +# open_how, so it refuses openat2 outright (ENOSYS), and every instancefs |
| +# open in here is an openat2 (#126, first roll; systemd 257). What it |
| +# guarded is covered: with no capabilities an instance can only make a |
| +# set-id file owned by carlos, which gains nothing for another carlos |
| +# process, NoNewPrivileges= voids it in here, and no other uid can reach |
| +# one (the image is nosuid by RootImageOptions=, the tenant trees are |
| +# nosuid, and /tmp is a nosuid tmpfs of this unit's own). |
| +# |
| +# Kept in the block for every template, the probe included (R11, rootfs |
| +# apps design §5.11). A release whose root-staged manifest declares jit |
| +# runs without it through a per-start drop-in that root writes and checks |
| +# the effective property of (C10), never through a unit file. The probe's |
| +# mdwe check: an mmap(PROT_WRITE|PROT_EXEC) answers EPERM inside the |
| +# probe, which never gets the drop-in. |
| +MemoryDenyWriteExecute=yes |
| +SystemCallArchitectures=native |
| + |
| +# --- per-service resources; the tier lives on the host slice (§4.3) --- |
| +TasksMax=256 |
| +# The kernel picks a tenant before it picks anything else in carlos.slice. |
| +OOMScoreAdjust=500 |
| +# One OOM kill takes every process of THIS service (systemd sets |
| +# memory.oom.group on the service cgroup for OOMPolicy=kill). A worker is |
| +# a sibling service with its own family rule (isolated-exec design §4.3; |
| +# rootfs apps design §5.4). |
| +OOMPolicy=kill |
diff --git a/internal/host/units_isolation_test.go b/internal/host/units_isolation_test.go| index 21745e8a3..daa41eeb6 100644 |
| --- a/internal/host/units_isolation_test.go |
| +++ b/internal/host/units_isolation_test.go |
| @@ -342,7 +342,8 @@ func TestControlUnitsShape(t *testing.T) { |
| if !have["carlos-instance-control.socket"] { |
| t.Error("enabledUnits lacks the control socket") |
| } |
| - if have["carlos-instance-control.service"] || have["carlos-instance-probe.service"] || have["carlos-instance@.service"] || have["carlos-instance-gen@.service"] { |
| + if have["carlos-instance-control.service"] || have["carlos-instance-probe.service"] || have["carlos-instance@.service"] || have["carlos-instance-gen@.service"] || |
| + have["carlos-instance-rootfs-probe.service"] || have["carlos-instance-rootfs@.service"] { |
| t.Error("a template, the probe or the socket-activated service must not be enabled") |
| } |
| for _, want := range []string{"d /data/.instances/_probe 0700 carlos carlos -", "d /run/carlos/.instances/_probe 2750 carlos carlos-sockets -"} { |
diff --git a/internal/host/units_rootfs_test.go b/internal/host/units_rootfs_test.go| new file mode 100644 |
| index 000000000..f95634363 |
| --- /dev/null |
| +++ b/internal/host/units_rootfs_test.go |
| @@ -0,0 +1,192 @@ |
| +package host |
| + |
| +import ( |
| + "strings" |
| + "testing" |
| +) |
| + |
| +var rootfsTemplates = []string{"carlos-instance-rootfs@.service", "carlos-instance-rootfs-gen@.service", "carlos-instance-rootfs-worker@.service", "carlos-instance-rootfs-cmd@.service", "carlos-instance-rootfs-probe.service"} |
| + |
| +func rootfsBlock(t *testing.T, name string) string { |
| + t.Helper() |
| + u := readUnit(t, name) |
| + start := strings.Index(u, "# --- the rootfs namespace") |
| + end := strings.Index(u, "# --- per-service resources") |
| + if start < 0 || end < 0 || end < start { |
| + t.Fatalf("%s: rootfs namespace block markers missing", name) |
| + } |
| + return u[start:end] |
| +} |
| + |
| +// The five rootfs units carry one restated namespace block, byte for byte, |
| +// so a property the rootfs probe proves is the property a rootfs tenant |
| +// gets (spec §5.3, §8; the binary templates' TestInstanceTemplatesShareOneNamespace). |
| +func TestRootfsTemplatesShareOneNamespace(t *testing.T) { |
| + t.Parallel() |
| + ref := rootfsBlock(t, "carlos-instance-rootfs@.service") |
| + for _, name := range rootfsTemplates[1:] { |
| + if rootfsBlock(t, name) != ref { |
| + t.Errorf("%s rootfs namespace block differs from carlos-instance-rootfs@.service", name) |
| + } |
| + } |
| +} |
| + |
| +// The restatement, pinned as an exact diff against the binary block: every |
| +// directive the binary block has is either kept or in the dropped set, and |
| +// every directive the rootfs block adds is in the added set. Any other |
| +// difference is a change nobody argued for. |
| +func TestRootfsNamespaceRestatesTheBinaryBlock(t *testing.T) { |
| + t.Parallel() |
| + binary := readUnit(t, "carlos-instance@.service") |
| + bstart, bend := strings.Index(binary, "# --- the namespace"), strings.Index(binary, "# --- per-service resources") |
| + bin := map[string]bool{} |
| + for _, d := range directives(binary[bstart:bend]) { |
| + bin[d] = true |
| + } |
| + rfs := map[string]bool{} |
| + for _, d := range directives(rootfsBlock(t, "carlos-instance-rootfs@.service")) { |
| + rfs[d] = true |
| + } |
| + dropped := map[string]string{ |
| + "TemporaryFileSystem=/var/cache:ro": "the host's /var is not in an image root (probe root-image, mounts-allowlisted)", |
| + "TemporaryFileSystem=/var/lib:ro": "same", |
| + "InaccessiblePaths=/etc/carlos /run/carlos-control": "neither path exists in an image root (probe etc-carlos, control-unreachable)", |
| + "PrivateTmp=yes": "replaced by a tmpfs on /tmp (probe tmp-private)", |
| + "ProtectControlGroups=yes": "replaced by strict (probe cgroup-strict)", |
| + "BindReadOnlyPaths=-/run/systemd/resolve/resolv.conf:/etc/resolv.conf": "replaced by root's per-instance copy outside the block (probe dns)", |
| + } |
| + added := map[string]bool{ |
| + "RootImageOptions=ro,nosuid,nodev": true, |
| + "MountAPIVFS=yes": true, |
| + "TemporaryFileSystem=/tmp:mode=1777,nosuid,nodev": true, |
| + "ProtectControlGroups=strict": true, |
| + "TemporaryFileSystem=/dev/pts:ro": true, // deviation 8; once Plan 6 adds it to the binary block it is kept, and Plan 6 drops this entry |
| + } |
| + for d := range bin { |
| + if rfs[d] { |
| + continue |
| + } |
| + if _, ok := dropped[d]; !ok { |
| + t.Errorf("binary block line %q is missing from the rootfs block and is not in the dropped table", d) |
| + } |
| + } |
| + for d := range dropped { |
| + if !bin[d] { |
| + t.Errorf("dropped table names %q, which the binary block no longer has; update the restatement", d) |
| + } |
| + if rfs[d] { |
| + t.Errorf("rootfs block still carries %q, which the table drops", d) |
| + } |
| + } |
| + for d := range rfs { |
| + if bin[d] || added[d] { |
| + continue |
| + } |
| + t.Errorf("rootfs block adds %q, which the restatement table does not name", d) |
| + } |
| + for d := range added { |
| + if !rfs[d] { |
| + t.Errorf("rootfs block lacks %q", d) |
| + } |
| + } |
| + // The egress fence is the binary block's, byte for byte. |
| + if !hasDirective(rootfsBlock(t, "carlos-instance-rootfs@.service"), instanceDeny) { |
| + t.Error("rootfs block's IPAddressDeny= differs from the binary block's") |
| + } |
| + // R11: the block itself keeps MemoryDenyWriteExecute=yes for every |
| + // template, the probe included. A jit release loses it only through |
| + // the per-start drop-in root writes (Task 5), never in a unit file. |
| + for _, name := range rootfsTemplates { |
| + if !hasDirective(rootfsBlock(t, name), "MemoryDenyWriteExecute=yes") { |
| + t.Errorf("%s block lacks MemoryDenyWriteExecute=yes", name) |
| + } |
| + } |
| +} |
| + |
| +// C3 in the unit files: every bind source a rootfs unit names is the |
| +// instance's own two directories, the platform binary, or root's exe |
| +// layout for that unit. Nothing from the version store, the live links or |
| +// another instance can be bound, whatever the control service writes. |
| +func TestRootfsTemplatesBindOnlyTheInstancesOwnAndRoots(t *testing.T) { |
| + t.Parallel() |
| + allowed := func(src, name string) bool { |
| + own := "%I" |
| + if name == "carlos-instance-rootfs-probe.service" { |
| + own = "_probe" |
| + } |
| + switch { |
| + case src == "/data/instances/"+own, src == "/run/carlos/instances/"+own, src == "/opt/carlos/carlos": |
| + return true |
| + case strings.HasPrefix(src, "/run/carlos-control/exe/"+own+"/rootfs/"): |
| + return true |
| + } |
| + return false |
| + } |
| + for _, name := range rootfsTemplates { |
| + for _, d := range directives(readUnit(t, name)) { |
| + for _, key := range []string{"BindPaths=", "BindReadOnlyPaths=", "RootImage="} { |
| + if !strings.HasPrefix(d, key) { |
| + continue |
| + } |
| + for _, spec := range strings.Fields(strings.TrimPrefix(d, key)) { |
| + src, _, _ := strings.Cut(strings.TrimPrefix(spec, "-"), ":") |
| + if !allowed(src, name) { |
| + t.Errorf("%s binds %q, outside the instance's own paths and root's exe layout", name, src) |
| + } |
| + } |
| + } |
| + if strings.Contains(d, "/opt/carlos/versions") || strings.Contains(d, "/opt/carlos/live") || strings.Contains(d, "/opt/carlos/app") { |
| + t.Errorf("%s names a build path: %q (the image is the app)", name, d) |
| + } |
| + } |
| + } |
| +} |
| + |
| +func TestRootfsTemplatesPerUnitLines(t *testing.T) { |
| + t.Parallel() |
| + for name, wants := range map[string][]string{ |
| + "carlos-instance-rootfs@.service": {"Type=simple", "ExecStart=/opt/carlos/carlos instance-exec --rootfs run %I", "ExecStopPost=/opt/carlos/carlos instance-exit --rootfs run %I", |
| + "RootImage=/run/carlos-control/exe/%I/rootfs/run/image", |
| + "BindReadOnlyPaths=/run/carlos-control/exe/%I/rootfs/run/launch.json:/opt/carlos/launch.json /run/carlos-control/exe/%I/rootfs/run/resolv.conf:/etc/resolv.conf", |
| + "BindPaths=/data/instances/%I /run/carlos/instances/%I", "WorkingDirectory=/data/instances/%I", "StandardOutput=file:/run/carlos/logs/%I.run.sock", |
| + "Slice=carlos-instances-%i.slice", "Restart=no", "KillMode=mixed", "TimeoutStopSec=20", "BindsTo=carlos-edge.service", |
| + "Requires=data-instances.mount run-carlos-instances.mount run-carlos-logs.mount", "SuccessExitStatus=129 130 141 143", "TasksMax=256", "OOMScoreAdjust=500", "OOMPolicy=kill", "User=carlos"}, |
| + "carlos-instance-rootfs-gen@.service": {"Type=simple", "ExecStart=/opt/carlos/carlos instance-exec --rootfs run %I", "ExecStopPost=/opt/carlos/carlos instance-exit --rootfs run %I", "WorkingDirectory=/data/instances", "SuccessExitStatus=129 130 141 143", "Restart=no", "User=carlos"}, |
| + "carlos-instance-rootfs-worker@.service": {"Type=simple", "ExecStart=/opt/carlos/carlos instance-exec --rootfs worker %I", "ExecStopPost=/opt/carlos/carlos instance-exit --rootfs worker %I", "WorkingDirectory=/data/instances", "SuccessExitStatus=129 130 141 143", "Restart=no", "BindsTo=carlos-edge.service", "User=carlos"}, |
| + "carlos-instance-rootfs-cmd@.service": {"Type=oneshot", "RemainAfterExit=yes", "TimeoutStartSec=6h5m", "ExecStart=/opt/carlos/carlos instance-exec --rootfs cmd %I", "ExecStopPost=/opt/carlos/carlos instance-exit --rootfs cmd %I", |
| + "RootImage=/run/carlos-control/exe/%I/rootfs/cmd/image", "StandardOutput=file:/run/carlos/logs/%I.cmd.sock", "StandardError=file:/run/carlos/logs/%I.cmd-err.sock", "WorkingDirectory=/data/instances/%I", "User=carlos"}, |
| + "carlos-instance-rootfs-probe.service": {"Type=oneshot", "ExecStart=/opt/carlos/carlos instance-probe --rootfs", "RootImage=/run/carlos-control/exe/_probe/rootfs/run/image", |
| + "BindPaths=/data/instances/_probe /run/carlos/instances/_probe", "StandardOutput=truncate:/run/carlos-control/rootfs-probe.out", "Slice=carlos-instances.slice", "TasksMax=64", "MemoryMax=64M", "CPUQuota=50%", "CPUWeight=50", "IOWeight=50", "User=carlos", |
| + "BindReadOnlyPaths=/run/carlos-control/exe/_probe/rootfs/run/launch.json:/opt/carlos/launch.json /run/carlos-control/exe/_probe/rootfs/run/resolv.conf:/etc/resolv.conf /run/carlos-control/exe/_probe/rootfs/run/box-addrs:/opt/carlos/box-addrs"}, |
| + } { |
| + u := readUnit(t, name) |
| + for _, w := range wants { |
| + if !hasDirective(u, w) { |
| + t.Errorf("%s lacks %q", name, w) |
| + } |
| + } |
| + if strings.Contains(u, "ExecStopPost=+") { |
| + t.Errorf("%s uses ExecStopPost=+ (codex R3-2)", name) |
| + } |
| + if name == "carlos-instance-rootfs-cmd@.service" && strings.Contains(u, "\nSuccessExitStatus=") { |
| + t.Errorf("%s counts a signal's exit as success", name) |
| + } |
| + } |
| + // The compound-instance templates key nothing by their instance string |
| + // inside the namespace or anywhere by %i (zero-downtime plan 2 D1). |
| + for _, name := range []string{"carlos-instance-rootfs-gen@.service", "carlos-instance-rootfs-worker@.service"} { |
| + for _, d := range directives(readUnit(t, name)) { |
| + for _, key := range []string{"Slice=", "BindPaths=", "ReadWritePaths=", "StandardOutput=", "RootImage="} { |
| + if strings.HasPrefix(d, key) { |
| + t.Errorf("%s carries %q; it belongs in the drop-in", name, d) |
| + } |
| + } |
| + if strings.HasPrefix(d, "BindReadOnlyPaths=") && strings.Contains(d, "/run/carlos-control/exe") { |
| + t.Errorf("%s binds a per-instance source itself: %q", name, d) |
| + } |
| + if strings.Contains(d, "%i") || (strings.Contains(d, "%I") && !strings.HasPrefix(d, "ExecStart=") && !strings.HasPrefix(d, "ExecStopPost=") && !strings.HasPrefix(d, "Description=")) { |
| + t.Errorf("%s keys %q by its instance string", name, d) |
| + } |
| + } |
| + } |
| +} |