amadan

carlos / platform Public

cgroup: the placement exec-shim (#272)

carlos-edge.service's RestrictNamespaces=yes makes systemd's seccomp
filter answer clone3 with ENOSYS, so CLONE_INTO_CGROUP placement fails
on every real box. The child now starts as the carlos binary itself
(/proc/self/exe __cgexec), writes its own PID into the target cgroup
through a directory fd the parent validated, verifies where it landed,
answers placed, and execs the tenant only once released. The tenant's
path, argv and env travel over a pipe; R is written immediately before
execve, so a start is reported failed only when non-execution is proven.

Adds startViaShim (not yet wired into startIn: Task 2), the in-flight
placement registry, the __cgprobe verb, and the hook as the first
statement of main() and of the package's TestMain.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude ff4b814a12c7d6dbf21906a5867b21690d27945b parent a5193cf
11 files changed, +2010 −0
  • internal/cgroup/cgroup.go +6 −0
  • internal/cgroup/inflight.go +104 −0
  • internal/cgroup/inflight_linux_test.go +93 −0
  • internal/cgroup/main_test.go +11 −0
  • internal/cgroup/shim.go +213 −0
  • internal/cgroup/shim_kernel_linux_test.go +694 −0
  • internal/cgroup/shim_linux.go +554 −0
  • internal/cgroup/shim_linux_test.go +236 −0
  • internal/cgroup/shim_other.go +20 −0
  • internal/cgroup/shim_test.go +74 −0
  • main.go +5 −0
diff --git a/internal/cgroup/cgroup.go b/internal/cgroup/cgroup.go
index bd3ad6953..839a29483 100644
--- a/internal/cgroup/cgroup.go
+++ b/internal/cgroup/cgroup.go
@@ -181,6 +181,12 @@ type Manager struct {
// locks holds each host's start lock (hostLock), made on first use.
locksMu sync.Mutex
locks map[string]*sync.Mutex
+ // placing is the in-flight registry (inflight.go): placements by
+ // cleaned target directory. placeWake is closed and dropped whenever a
+ // count drops, so waitPlacementsUnder wakes and rechecks.
+ placeMu sync.Mutex
+ placing map[string]int
+ placeWake chan struct{}
}
// hostLock is host's start lock. A start holds it while it checks for, and
diff --git a/internal/cgroup/inflight.go b/internal/cgroup/inflight.go
new file mode 100644
index 000000000..fa955bf90
--- /dev/null
+++ b/internal/cgroup/inflight.go
@@ -0,0 +1,104 @@
+package cgroup
+
+import (
+ "context"
+ "path/filepath"
+ "strings"
+ "sync"
+)
+
+// The in-flight registry: target cgroup directories that a shim is being
+// placed into, from just before its fork until it answers placed or the
+// start fails. With clone3 a child appeared in its cgroup atomically; a
+// shim spends a few milliseconds in the agent's cgroup first, so a proof
+// that a cgroup is empty must count these too (Ruling 6, Task 2).
+
+// beginPlacement registers one placement into dir. done ends it, and is
+// idempotent: a second call does nothing. A nil manager registers nothing.
+func (m *Manager) beginPlacement(dir string) (done func()) {
+ if m == nil {
+ return func() {}
+ }
+ dir = filepath.Clean(dir)
+ m.placeMu.Lock()
+ if m.placing == nil {
+ m.placing = make(map[string]int)
+ }
+ m.placing[dir]++
+ m.placeMu.Unlock()
+ var once sync.Once
+ return func() {
+ once.Do(func() {
+ m.placeMu.Lock()
+ defer m.placeMu.Unlock()
+ if m.placing[dir]--; m.placing[dir] <= 0 {
+ delete(m.placing, dir)
+ }
+ // Wake every waiter; each rechecks its own subtree.
+ if m.placeWake != nil {
+ close(m.placeWake)
+ m.placeWake = nil
+ }
+ })
+ }
+}
+
+// placementsUnder counts placements into dir or anywhere below it.
+func (m *Manager) placementsUnder(dir string) int {
+ if m == nil {
+ return 0
+ }
+ m.placeMu.Lock()
+ defer m.placeMu.Unlock()
+ return m.countUnderLocked(filepath.Clean(dir))
+}
+
+// placementsAt counts placements into dir itself.
+func (m *Manager) placementsAt(dir string) int {
+ if m == nil {
+ return 0
+ }
+ m.placeMu.Lock()
+ defer m.placeMu.Unlock()
+ return m.placing[filepath.Clean(dir)]
+}
+
+// waitPlacementsUnder returns nil once no placement is in flight in dir or
+// below it, or ctx's error if ctx ends first.
+func (m *Manager) waitPlacementsUnder(ctx context.Context, dir string) error {
+ if m == nil {
+ return nil
+ }
+ dir = filepath.Clean(dir)
+ for {
+ m.placeMu.Lock()
+ if m.countUnderLocked(dir) == 0 {
+ m.placeMu.Unlock()
+ return nil
+ }
+ if m.placeWake == nil {
+ m.placeWake = make(chan struct{})
+ }
+ wake := m.placeWake
+ m.placeMu.Unlock()
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ case <-wake:
+ }
+ }
+}
+
+func (m *Manager) countUnderLocked(dir string) int {
+ prefix := dir
+ if !strings.HasSuffix(prefix, "/") {
+ prefix += "/"
+ }
+ n := 0
+ for target, c := range m.placing {
+ if target == dir || strings.HasPrefix(target, prefix) {
+ n += c
+ }
+ }
+ return n
+}
diff --git a/internal/cgroup/inflight_linux_test.go b/internal/cgroup/inflight_linux_test.go
new file mode 100644
index 000000000..f6253fe22
--- /dev/null
+++ b/internal/cgroup/inflight_linux_test.go
@@ -0,0 +1,93 @@
+//go:build linux
+
+package cgroup
+
+import (
+ "context"
+ "errors"
+ "testing"
+ "time"
+)
+
+func TestPlacementsUnderCountsTheSubtree(t *testing.T) {
+ m := &Manager{root: "/r", logf: t.Logf}
+ done := m.beginPlacement("/r/carlos-exec@a.example/g2")
+ if m.placementsUnder("/r/carlos-exec@a.example") != 1 || m.placementsAt("/r/carlos-exec@a.example") != 0 {
+ t.Fatal("a child's placement is under its parent, not at it")
+ }
+ if m.placementsUnder("/r/carlos-exec@a.exampleX") != 0 {
+ t.Fatal("a prefix without a separator matched")
+ }
+ if m.placementsAt("/r/carlos-exec@a.example/g2/") != 1 || m.placementsUnder("/r") != 1 {
+ t.Fatal("the target itself or its root does not count it")
+ }
+ done()
+ done() // idempotent
+ if m.placementsUnder("/r/carlos-exec@a.example") != 0 {
+ t.Fatal("done did not drop the count")
+ }
+}
+
+func TestPlacementsCountEachInFlight(t *testing.T) {
+ m := &Manager{root: "/r", logf: t.Logf}
+ a := m.beginPlacement("/r/h")
+ b := m.beginPlacement("/r/h")
+ if m.placementsAt("/r/h") != 2 {
+ t.Fatal("two placements at one target count as one")
+ }
+ a()
+ a()
+ if m.placementsAt("/r/h") != 1 {
+ t.Fatal("a repeated done dropped another placement's count")
+ }
+ b()
+ if m.placementsAt("/r/h") != 0 {
+ t.Fatal("the count did not reach zero")
+ }
+}
+
+func TestWaitPlacementsUnderWakes(t *testing.T) {
+ m := &Manager{root: "/r", logf: t.Logf}
+ if err := m.waitPlacementsUnder(context.Background(), "/r"); err != nil {
+ t.Fatalf("nothing in flight: %v", err)
+ }
+ done := m.beginPlacement("/r/h/g1")
+
+ ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond)
+ defer cancel()
+ if err := m.waitPlacementsUnder(ctx, "/r/h"); !errors.Is(err, context.DeadlineExceeded) {
+ t.Fatalf("err=%v, want context.DeadlineExceeded", err)
+ }
+
+ got := make(chan error, 1)
+ go func() { got <- m.waitPlacementsUnder(context.Background(), "/r/h") }()
+ // An unrelated placement ending wakes the waiter, which must keep waiting.
+ other := m.beginPlacement("/r/elsewhere")
+ other()
+ select {
+ case err := <-got:
+ t.Fatalf("the waiter returned %v while a placement was in flight", err)
+ case <-time.After(50 * time.Millisecond):
+ }
+ done()
+ select {
+ case err := <-got:
+ if err != nil {
+ t.Fatal(err)
+ }
+ case <-time.After(time.Second):
+ t.Fatal("the waiter did not wake within 1s of done")
+ }
+}
+
+func TestNilManagerPlacements(t *testing.T) {
+ var m *Manager
+ done := m.beginPlacement("/r/h")
+ done()
+ if m.placementsUnder("/r") != 0 || m.placementsAt("/r/h") != 0 {
+ t.Fatal("a nil manager counted a placement")
+ }
+ if err := m.waitPlacementsUnder(context.Background(), "/r"); err != nil {
+ t.Fatal(err)
+ }
+}
diff --git a/internal/cgroup/main_test.go b/internal/cgroup/main_test.go
new file mode 100644
index 000000000..16bf7f65a
--- /dev/null
+++ b/internal/cgroup/main_test.go
@@ -0,0 +1,11 @@
+package cgroup
+
+import (
+ "os"
+ "testing"
+)
+
+func TestMain(m *testing.M) {
+ RunShimIfRequested()
+ os.Exit(m.Run())
+}
diff --git a/internal/cgroup/shim.go b/internal/cgroup/shim.go
new file mode 100644
index 000000000..a02bca75c
--- /dev/null
+++ b/internal/cgroup/shim.go
@@ -0,0 +1,213 @@
+package cgroup
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "os"
+ "strconv"
+ "strings"
+ "sync/atomic"
+ "time"
+)
+
+// ShimVerb is the carlos binary's hidden placement verb (Paul's ruling,
+// #272, 2026-10-13). carlos-edge.service's RestrictNamespaces=yes makes
+// systemd's seccomp filter answer clone3 with ENOSYS, so no child can be
+// created inside its cgroup. Instead the child starts as this binary,
+// writes its own PID into the target cgroup through a directory fd the
+// parent validated, checks where it landed, and only then execs the
+// tenant: the tenant's code never runs outside its limits.
+const ShimVerb = "__cgexec"
+
+// ProbeVerb prints the process's own cgroup and exits: the tenant the
+// arm-time placement check (VerifyPlacement) execs through the shim.
+const ProbeVerb = "__cgprobe"
+
+var shimHooked atomic.Bool
+
+// RunShimIfRequested must be the first statement of main(), and of TestMain
+// in a test binary that places processes. It marks this process as able to
+// serve the shim (shimAvailable), and when this process IS a shim or a
+// probe it runs it and exits; it returns only when it is neither.
+func RunShimIfRequested() {
+ shimHooked.Store(true)
+ if len(os.Args) < 2 {
+ return
+ }
+ switch os.Args[1] {
+ case ShimVerb:
+ os.Exit(runShim(os.Args[2:]))
+ case ProbeVerb:
+ os.Exit(runProbe())
+ }
+}
+
+// placementError is a start whose process could not be placed in its
+// cgroup before exec. Nothing of the tenant ran.
+type placementError struct{ err error }
+
+func (e *placementError) Error() string { return "placement before exec: " + e.err.Error() }
+func (e *placementError) Unwrap() error { return e.err }
+
+func isPlacementError(err error) bool {
+ var p *placementError
+ return errors.As(err, &p)
+}
+
+// ErrPlacementInFlight is a cleanup asked for members while a process is
+// being placed in its cgroup (Task 2).
+var ErrPlacementInFlight = errors.New("cgroup: a process is being placed in this cgroup")
+
+// shimDeadline bounds each of the parent's two waits on the shim: start to
+// placement, and release to exec (Ruling 9).
+var shimDeadline = 10 * time.Second
+
+// shimExe is what the parent starts as the shim: the running build, even
+// after a roll has replaced the file on disk.
+var shimExe = "/proc/self/exe"
+
+// shimTestHold is copied into each spec's TestHold. Only test files set it,
+// and only a test binary's shim honours it.
+var shimTestHold int
+
+// shimAfterRelease, when set, runs between the release byte and the read of
+// the exec status. Only test files set it.
+var shimAfterRelease func()
+
+// shimSpec is what the parent sends the shim over the spec pipe: never in
+// the shim's argv or environment, so no secret shows in /proc/<pid>/cmdline
+// and no tenant variable shapes the shim before placement.
+type shimSpec struct {
+ Path string
+ Dir string // the tenant's cmd.Dir; the shim changes to it after placement
+ Argv []string
+ Env []string
+ Expected string // "/..." as /proc/self/cgroup shows it after "0::"
+ KeepFDs int // fds 0..KeepFDs-1 are the tenant's
+ TestHold int // test binaries only: 0 none, 1 hold before step 4, 2 hold after step 10 (before R)
+}
+
+const (
+ shimSpecMagic = "carlos-cgexec-1"
+ // shimSpecMax bounds the encoded spec the shim will read.
+ shimSpecMax = 16 << 20
+ // shimSpecMaxCount bounds each decoded count (argv, env, KeepFDs).
+ shimSpecMaxCount = 1 << 20
+)
+
+// encodeShimSpec writes each field followed by a NUL, in order: the magic,
+// Path, Dir, Expected, KeepFDs, TestHold, len(Argv), each argv element,
+// len(Env), each env entry. Bytes pass through untouched (invalid UTF-8
+// included); a string holding a NUL is refused, as execve could not carry
+// it either.
+func encodeShimSpec(s shimSpec) ([]byte, error) {
+ var b []byte
+ put := func(field, v string) error {
+ if strings.IndexByte(v, 0) >= 0 {
+ return fmt.Errorf("cgroup: shim spec: %s contains a NUL byte", field)
+ }
+ b = append(b, v...)
+ b = append(b, 0)
+ return nil
+ }
+ if err := put("magic", shimSpecMagic); err != nil {
+ return nil, err
+ }
+ for _, f := range []struct{ name, v string }{
+ {"path", s.Path}, {"dir", s.Dir}, {"expected", s.Expected},
+ {"keep", strconv.Itoa(s.KeepFDs)}, {"hold", strconv.Itoa(s.TestHold)},
+ {"argc", strconv.Itoa(len(s.Argv))},
+ } {
+ if err := put(f.name, f.v); err != nil {
+ return nil, err
+ }
+ }
+ for _, a := range s.Argv {
+ if err := put("argv", a); err != nil {
+ return nil, err
+ }
+ }
+ if err := put("envc", strconv.Itoa(len(s.Env))); err != nil {
+ return nil, err
+ }
+ for _, e := range s.Env {
+ if err := put("env", e); err != nil {
+ return nil, err
+ }
+ }
+ return b, nil
+}
+
+// decodeShimSpec is encodeShimSpec's inverse. It refuses a wrong magic, a
+// buffer that ends early or runs past the last field, and a count above
+// shimSpecMaxCount.
+func decodeShimSpec(b []byte) (shimSpec, error) {
+ var s shimSpec
+ next := func() (string, error) {
+ i := bytes.IndexByte(b, 0)
+ if i < 0 {
+ return "", errors.New("cgroup: shim spec: truncated")
+ }
+ v := string(b[:i])
+ b = b[i+1:]
+ return v, nil
+ }
+ count := func() (int, error) {
+ v, err := next()
+ if err != nil {
+ return 0, err
+ }
+ n, err := strconv.Atoi(v)
+ if err != nil || n < 0 || n > shimSpecMaxCount {
+ return 0, fmt.Errorf("cgroup: shim spec: bad count %q", v)
+ }
+ return n, nil
+ }
+ magic, err := next()
+ if err != nil {
+ return s, err
+ }
+ if magic != shimSpecMagic {
+ return s, fmt.Errorf("cgroup: shim spec: magic %q, want %q", magic, shimSpecMagic)
+ }
+ for _, p := range []*string{&s.Path, &s.Dir, &s.Expected} {
+ if *p, err = next(); err != nil {
+ return s, err
+ }
+ }
+ if s.KeepFDs, err = count(); err != nil {
+ return s, err
+ }
+ if s.TestHold, err = count(); err != nil {
+ return s, err
+ }
+ argc, err := count()
+ if err != nil {
+ return s, err
+ }
+ s.Argv = []string{}
+ for range argc {
+ a, err := next()
+ if err != nil {
+ return s, err
+ }
+ s.Argv = append(s.Argv, a)
+ }
+ envc, err := count()
+ if err != nil {
+ return s, err
+ }
+ s.Env = []string{}
+ for range envc {
+ e, err := next()
+ if err != nil {
+ return s, err
+ }
+ s.Env = append(s.Env, e)
+ }
+ if len(b) != 0 {
+ return s, fmt.Errorf("cgroup: shim spec: %d trailing bytes", len(b))
+ }
+ return s, nil
+}
diff --git a/internal/cgroup/shim_kernel_linux_test.go b/internal/cgroup/shim_kernel_linux_test.go
new file mode 100644
index 000000000..d64dd4037
--- /dev/null
+++ b/internal/cgroup/shim_kernel_linux_test.go
@@ -0,0 +1,694 @@
+//go:build linux
+
+package cgroup
+
+import (
+ "bytes"
+ "errors"
+ "fmt"
+ "io"
+ "os"
+ "os/exec"
+ "os/signal"
+ "path/filepath"
+ "slices"
+ "strconv"
+ "strings"
+ "syscall"
+ "testing"
+ "time"
+ "unsafe"
+
+ "golang.org/x/sys/unix"
+)
+
+// The placement shim against a real kernel (zdd readiness 3, Task 1).
+// Opt-in: CARLOS_SHIM_KERNEL=1 inside a disposable unit with Delegate=yes:
+//
+// systemd-run --user --wait --pipe --collect -p Delegate=yes \
+// -E TMPDIR=/tmp/k/zddr3 -E CARLOS_SHIM_KERNEL=1 -E CARLOS_GEN_KERNEL=1 \
+// cgroup.test -test.run 'ShimKernel|ShimExecFailure' -test.v -test.count=1
+//
+// Without the variable every test here skips. With it, a missing delegation
+// fails the test: a run that was asked for cannot pass by skipping.
+
+// shimKernelManager is genKernelManager's setup, except that once
+// CARLOS_SHIM_KERNEL=1 asks for the run, no delegation is a failure.
+func shimKernelManager(t *testing.T) *Manager {
+ t.Helper()
+ if os.Getenv("CARLOS_SHIM_KERNEL") != "1" {
+ t.Skip("set CARLOS_SHIM_KERNEL=1 inside a unit with Delegate=yes (zdd readiness 3, Task 1)")
+ }
+ raw, err := os.ReadFile("/proc/self/cgroup")
+ if err != nil {
+ t.Fatalf("CARLOS_SHIM_KERNEL=1 but no cgroup: %v", err)
+ }
+ rel, ok := ParseSelfCgroup(string(raw))
+ if !ok {
+ t.Fatal("CARLOS_SHIM_KERNEL=1 but no unified cgroup path")
+ }
+ root := filepath.Join("/sys/fs/cgroup", rel)
+ if filepath.Base(root) == "supervise" {
+ root = filepath.Dir(root) // an earlier test in this process moved us
+ }
+ sup := filepath.Join(root, "supervise")
+ if err := os.MkdirAll(sup, 0o755); err != nil {
+ t.Fatalf("CARLOS_SHIM_KERNEL=1 but no delegation: %v", err)
+ }
+ if err := os.WriteFile(filepath.Join(sup, "cgroup.procs"), []byte(strconv.Itoa(os.Getpid())), 0o644); err != nil {
+ t.Fatalf("CARLOS_SHIM_KERNEL=1 but no delegation: %v", err)
+ }
+ if err := enableChildren(root); err != nil {
+ t.Fatalf("CARLOS_SHIM_KERNEL=1 but no delegation: %v", err)
+ }
+ return &Manager{root: root, logf: t.Logf}
+}
+
+// shimKernelTarget makes m.root/shim-<name> with the default tier's limits,
+// and removes it when the test ends.
+func shimKernelTarget(t *testing.T, m *Manager, name string) string {
+ t.Helper()
+ dir := filepath.Join(m.root, "shim-"+name)
+ if err := os.Mkdir(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ if err := os.Remove(dir); err != nil {
+ t.Errorf("remove %s: %v", dir, err)
+ }
+ })
+ if err := m.applyLimit(name, dir, TierDefault, true); err != nil {
+ t.Fatal(err)
+ }
+ return dir
+}
+
+// markerCmd is a tenant that touches marker first thing, then runs script.
+func markerCmd(marker, script string) *exec.Cmd {
+ return exec.Command("/bin/sh", "-c", `touch "$0"; `+script, marker)
+}
+
+func assertNoMarker(t *testing.T, marker string) {
+ t.Helper()
+ if _, err := os.Stat(marker); err == nil {
+ t.Fatal("tenant code ran")
+ }
+}
+
+func assertReaped(t *testing.T, cmd *exec.Cmd) {
+ t.Helper()
+ if cmd.Process == nil {
+ t.Fatal("no shim was started")
+ }
+ if _, err := os.Stat(fmt.Sprintf("/proc/%d", cmd.Process.Pid)); err == nil {
+ t.Fatalf("pid %d was not reaped", cmd.Process.Pid)
+ }
+}
+
+func assertNothingInFlight(t *testing.T, m *Manager) {
+ t.Helper()
+ if n := m.placementsUnder(m.root); n != 0 {
+ t.Fatalf("%d placements still registered", n)
+ }
+}
+
+func shimCgroupPath(dir string) string { return strings.TrimPrefix(dir, "/sys/fs/cgroup") }
+
+// setShimVar sets a package var for one test.
+func setShimVar[T any](t *testing.T, p *T, v T) {
+ orig := *p
+ *p = v
+ t.Cleanup(func() { *p = orig })
+}
+
+// shimDumpSep separates the dump's argv, env and cwd. It cannot be "\x00\x00":
+// an empty argv element joins to exactly that.
+const shimDumpSep = "\x00--shim-dump--\x00"
+
+// TestShimKernelChild is the synthetic tenant. Outside a child run it does
+// nothing, so a kernel run shows no skip for it.
+func TestShimKernelChild(t *testing.T) {
+ switch os.Getenv("CARLOS_SHIM_CHILD") {
+ case "dump":
+ wd, err := os.Getwd()
+ if err != nil {
+ os.Exit(3)
+ }
+ var b bytes.Buffer
+ b.WriteString(strings.Join(os.Args, "\x00"))
+ b.WriteString(shimDumpSep)
+ b.WriteString(strings.Join(os.Environ(), "\x00"))
+ b.WriteString(shimDumpSep)
+ b.WriteString(wd)
+ os.Stdout.Write(b.Bytes())
+ os.Exit(0)
+ case "state":
+ var sig int
+ if err := unix.Prctl(unix.PR_GET_PDEATHSIG, uintptr(unsafe.Pointer(&sig)), 0, 0, 0); err != nil {
+ os.Exit(3)
+ }
+ pgid, err := unix.Getpgid(0)
+ if err != nil {
+ os.Exit(3)
+ }
+ fmt.Printf("pdeathsig %d\npgid-leader %v\n", sig, pgid == os.Getpid())
+ os.Exit(0)
+ }
+}
+
+func TestShimKernelPlacesBeforeExec(t *testing.T) {
+ m := shimKernelManager(t)
+ dir := shimKernelTarget(t, m, "places")
+ cmd := exec.Command("/bin/sh", "-c", `cat /proc/self/cgroup; echo $$; read x || :`)
+ var out bytes.Buffer
+ cmd.Stdout = &out
+ stdin, err := cmd.StdinPipe()
+ if err != nil {
+ t.Fatal(err)
+ }
+ refused, err := m.startViaShim(dir, cmd, nil)
+ if refused || err != nil {
+ t.Fatalf("refused=%v err=%v", refused, err)
+ }
+ assertNothingInFlight(t, m)
+ want := "0::" + shimCgroupPath(dir)
+ raw, err := os.ReadFile(fmt.Sprintf("/proc/%d/cgroup", cmd.Process.Pid))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if strings.TrimSpace(string(raw)) != want {
+ t.Fatalf("/proc/%d/cgroup = %q, want %q", cmd.Process.Pid, raw, want)
+ }
+ stdin.Close()
+ if err := cmd.Wait(); err != nil {
+ t.Fatal(err)
+ }
+ lines := strings.Split(strings.TrimSpace(out.String()), "\n")
+ if len(lines) != 2 || lines[0] != want || lines[1] != strconv.Itoa(cmd.Process.Pid) {
+ t.Fatalf("tenant printed %q, want %q and pid %d", out.String(), want, cmd.Process.Pid)
+ }
+}
+
+func TestShimKernelExitCodePassesThrough(t *testing.T) {
+ m := shimKernelManager(t)
+ dir := shimKernelTarget(t, m, "exitcode")
+ cmd := exec.Command("/bin/sh", "-c", "exit 7")
+ if _, err := m.startViaShim(dir, cmd, nil); err != nil {
+ t.Fatal(err)
+ }
+ err := cmd.Wait()
+ var ee *exec.ExitError
+ if !errors.As(err, &ee) || ee.ExitCode() != 7 {
+ t.Fatalf("Wait = %v, want exit status 7", err)
+ }
+}
+
+func TestShimKernelPassesArgvEnvExactly(t *testing.T) {
+ m := shimKernelManager(t)
+ dir := shimKernelTarget(t, m, "argvenv")
+ wd := t.TempDir()
+ mk := func(large string) *exec.Cmd {
+ return &exec.Cmd{
+ Path: os.Args[0],
+ Args: []string{os.Args[0], "-test.run=^TestShimKernelChild$", "", "two words", "line\nbreak"},
+ Env: []string{"CARLOS_SHIM_CHILD=dump", "EMPTY=", "EQ=a=b", "BAD=\xff\xfe", "LARGE=" + large},
+ Dir: wd,
+ }
+ }
+ run := func(cmd *exec.Cmd, viaShim bool) []byte {
+ t.Helper()
+ var out bytes.Buffer
+ cmd.Stdout = &out
+ cmd.Stderr = os.Stderr
+ var err error
+ if viaShim {
+ _, err = m.startViaShim(dir, cmd, nil)
+ } else {
+ err = cmd.Start()
+ }
+ if err != nil {
+ t.Fatalf("start (shim %v): %v", viaShim, err)
+ }
+ if err := cmd.Wait(); err != nil {
+ t.Fatalf("wait (shim %v): %v", viaShim, err)
+ }
+ return out.Bytes()
+ }
+ large := strings.Repeat("v", 100<<10)
+ want := mk(large)
+ viaShim := run(mk(large), true)
+ direct := run(mk(large), false)
+ if !bytes.Equal(viaShim, direct) {
+ t.Fatalf("the shim's tenant saw different argv, env or cwd from a direct start's")
+ }
+ parts := bytes.Split(viaShim, []byte(shimDumpSep))
+ if len(parts) != 3 {
+ t.Fatalf("dump has %d parts", len(parts))
+ }
+ if got := strings.Split(string(parts[0]), "\x00"); !slices.Equal(got, want.Args) {
+ t.Fatalf("argv = %q, want %q", got, want.Args)
+ }
+ if got := strings.Split(string(parts[1]), "\x00"); !slices.Equal(got, want.Env) {
+ t.Fatalf("env differs from cmd.Env (%d entries, want %d)", len(got), len(want.Env))
+ }
+ if string(parts[2]) != wd {
+ t.Fatalf("cwd = %q, want %q", parts[2], wd)
+ }
+
+ // Above Linux's per-string limit (32 pages): both fail with E2BIG, and
+ // the shim's is the command's own error, not a placement failure.
+ huge := strings.Repeat("v", 200<<10)
+ if err := mk(huge).Start(); !errors.Is(err, syscall.E2BIG) {
+ t.Fatalf("direct start = %v, want E2BIG", err)
+ }
+ cmd := mk(huge)
+ refused, err := m.startViaShim(dir, cmd, nil)
+ var pe *os.PathError
+ if refused || isPlacementError(err) || !errors.As(err, &pe) || pe.Op != "fork/exec" || !errors.Is(err, syscall.E2BIG) {
+ t.Fatalf("shim start = %v, want *os.PathError fork/exec E2BIG", err)
+ }
+ assertReaped(t, cmd)
+ assertNothingInFlight(t, m)
+}
+
+// stateScript prints the tenant's inherited state from a shell, which,
+// unlike a Go program, leaves its signal mask and RLIMIT_NOFILE as it found
+// them, then execs the Go child for what only prctl can read. ls lists the
+// shell's descriptors, not its own.
+const stateScript = `echo fds-begin; ls /proc/$$/fd; echo fds-end
+grep -E '^(SigBlk|SigIgn|CapInh|CapPrm|CapEff|CapBnd|CapAmb|NoNewPrivs):' /proc/$$/status
+echo nofile $(ulimit -Sn) $(ulimit -Hn)
+exec "$0" "$@"`
+
+func TestShimKernelInheritedStateMatchesADirectStart(t *testing.T) {
+ m := shimKernelManager(t)
+ dir := shimKernelTarget(t, m, "state")
+ signal.Ignore(syscall.SIGHUP)
+ t.Cleanup(func() { signal.Reset(syscall.SIGHUP) })
+ opened, err := syscall.Open("/dev/null", syscall.O_RDONLY, 0) // no O_CLOEXEC
+ if err != nil {
+ t.Fatal(err)
+ }
+ // Moved to 100 or above (F_DUPFD leaves close-on-exec clear), so the
+ // tenant's ExtraFiles at 3 and 4 cannot cover it in a direct start.
+ leaked, err := unix.FcntlInt(uintptr(opened), unix.F_DUPFD, 100)
+ syscall.Close(opened)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { syscall.Close(leaked) })
+
+ run := func(viaShim bool) (fds []string, rest []string) {
+ t.Helper()
+ r1, w1, err := os.Pipe()
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer r1.Close()
+ defer w1.Close()
+ r2, w2, err := os.Pipe()
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer r2.Close()
+ defer w2.Close()
+ cmd := exec.Command("/bin/sh", "-c", stateScript, os.Args[0], "-test.run=^TestShimKernelChild$")
+ cmd.Env = []string{"CARLOS_SHIM_CHILD=state"}
+ cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true, Pdeathsig: syscall.SIGKILL}
+ cmd.ExtraFiles = []*os.File{r1, w2}
+ var out bytes.Buffer
+ cmd.Stdout = &out
+ cmd.Stderr = os.Stderr
+ if viaShim {
+ _, err = m.startViaShim(dir, cmd, nil)
+ } else {
+ err = cmd.Start()
+ }
+ if err != nil {
+ t.Fatalf("start (shim %v): %v", viaShim, err)
+ }
+ if err := cmd.Wait(); err != nil {
+ t.Fatalf("wait (shim %v): %v", viaShim, err)
+ }
+ inFDs := false
+ for _, l := range strings.Split(strings.TrimSpace(out.String()), "\n") {
+ switch {
+ case l == "fds-begin":
+ inFDs = true
+ case l == "fds-end":
+ inFDs = false
+ case inFDs:
+ fds = append(fds, l)
+ default:
+ rest = append(rest, l)
+ }
+ }
+ return fds, rest
+ }
+ shimFDs, shimState := run(true)
+ directFDs, directState := run(false)
+ t.Logf("shim tenant: fds %v %q", shimFDs, shimState)
+ t.Logf("direct tenant: fds %v %q", directFDs, directState)
+
+ if !slices.Equal(shimFDs, []string{"0", "1", "2", "3", "4"}) {
+ t.Fatalf("the shim's tenant has fds %v, want exactly 0 1 2 3 4", shimFDs)
+ }
+ // The control: a direct start does inherit the leaked descriptor, so the
+ // shim's clean listing is the shim's doing and not an absent leak.
+ if !slices.Contains(directFDs, strconv.Itoa(leaked)) {
+ t.Fatalf("the direct start's fds %v lack the leaked fd %d: the control is broken", directFDs, leaked)
+ }
+ if !slices.Equal(shimState, directState) {
+ t.Fatalf("inherited state differs:\nshim %q\ndirect %q", shimState, directState)
+ }
+ // And the compared state is not vacuous.
+ for _, want := range []string{"pdeathsig 9", "pgid-leader true"} {
+ if !slices.Contains(shimState, want) {
+ t.Fatalf("shim state %q lacks %q", shimState, want)
+ }
+ }
+ for _, l := range shimState {
+ if v, ok := strings.CutPrefix(l, "SigIgn:\t"); ok {
+ mask, err := strconv.ParseUint(v, 16, 64)
+ if err != nil || mask&(1<<(uint(syscall.SIGHUP)-1)) == 0 {
+ t.Fatalf("SigIgn %q does not carry the ignored SIGHUP", v)
+ }
+ }
+ }
+}
+
+func TestShimKernelChdirFailureIsTheCommands(t *testing.T) {
+ m := shimKernelManager(t)
+ dir := shimKernelTarget(t, m, "chdir")
+ marker := filepath.Join(t.TempDir(), "ran")
+ cmd := markerCmd(marker, "")
+ missing := filepath.Join(t.TempDir(), "missing")
+ cmd.Dir = missing
+ refused, err := m.startViaShim(dir, cmd, nil)
+ var pe *os.PathError
+ if refused || isPlacementError(err) || !errors.As(err, &pe) || pe.Op != "chdir" || pe.Path != missing || !errors.Is(err, syscall.ENOENT) {
+ t.Fatalf("refused=%v err=%v, want *os.PathError chdir ENOENT", refused, err)
+ }
+ assertReaped(t, cmd)
+ assertNoMarker(t, marker)
+ assertNothingInFlight(t, m)
+}
+
+func TestShimExecFailureIsTheCommands(t *testing.T) {
+ m := shimKernelManager(t)
+ dir := shimKernelTarget(t, m, "execfail")
+ cmd := &exec.Cmd{Path: "/nonexistent/app", Args: []string{"app"}}
+ refused, err := m.startViaShim(dir, cmd, nil)
+ var pe *os.PathError
+ if refused || isPlacementError(err) || !errors.As(err, &pe) || pe.Op != "fork/exec" || pe.Path != "/nonexistent/app" || !errors.Is(err, syscall.ENOENT) {
+ t.Fatalf("refused=%v err=%v, want *os.PathError fork/exec ENOENT", refused, err)
+ }
+ t.Logf("the command's own error: %v", err)
+ assertReaped(t, cmd)
+ assertNothingInFlight(t, m)
+}
+
+func TestShimKernelPlacementFailureNeverRunsTenant(t *testing.T) {
+ m := shimKernelManager(t)
+ dir := shimKernelTarget(t, m, "noprocs")
+ procs := filepath.Join(dir, "cgroup.procs")
+ if err := os.Chmod(procs, 0o444); err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = os.Chmod(procs, 0o644) })
+ marker := filepath.Join(t.TempDir(), "ran")
+ cmd := markerCmd(marker, "")
+ refused, err := m.startViaShim(dir, cmd, nil)
+ if refused || !isPlacementError(err) || !strings.Contains(err.Error(), "procs") {
+ t.Fatalf("refused=%v err=%v, want the shim's procs refusal as a placement error", refused, err)
+ }
+ t.Logf("placement error: %v", err)
+ assertReaped(t, cmd)
+ assertNoMarker(t, marker)
+ assertNothingInFlight(t, m)
+}
+
+func TestShimKernelBeforeRunsAfterPlacement(t *testing.T) {
+ m := shimKernelManager(t)
+ dir := shimKernelTarget(t, m, "before")
+ var events string
+ inFlight := -1
+ before := func() error {
+ b, err := os.ReadFile(filepath.Join(dir, "cgroup.events"))
+ events = string(b)
+ inFlight = m.placementsUnder(dir)
+ return err
+ }
+ cmd := exec.Command("/bin/true")
+ if refused, err := m.startViaShim(dir, cmd, before); refused || err != nil {
+ t.Fatalf("refused=%v err=%v", refused, err)
+ }
+ if err := cmd.Wait(); err != nil {
+ t.Fatal(err)
+ }
+ if !slices.Contains(strings.Split(events, "\n"), "populated 1") {
+ t.Fatalf("before saw cgroup.events %q: the shim was not in place when it was asked", events)
+ }
+ if inFlight != 0 {
+ t.Fatalf("before ran with %d placements registered; the registry covers only fork to placement", inFlight)
+ }
+
+ refusal := errors.New("not yours any more")
+ marker := filepath.Join(t.TempDir(), "ran")
+ cmd = markerCmd(marker, "")
+ refused, err := m.startViaShim(dir, cmd, func() error { return refusal })
+ if !refused || err != refusal {
+ t.Fatalf("refused=%v err=%v, want before's own error", refused, err)
+ }
+ assertReaped(t, cmd)
+ assertNoMarker(t, marker)
+ assertNothingInFlight(t, m)
+}
+
+// waitShimHeld waits until the shim has marked its spec fd close-on-exec
+// (its step 3), which it does just before the step-1 test hold.
+func waitShimHeld(pid, specFD int) bool {
+ path := fmt.Sprintf("/proc/%d/fdinfo/%d", pid, specFD)
+ for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) {
+ b, err := os.ReadFile(path)
+ if err != nil {
+ continue
+ }
+ for _, l := range strings.Split(string(b), "\n") {
+ if v, ok := strings.CutPrefix(l, "flags:"); ok {
+ flags, err := strconv.ParseUint(strings.TrimSpace(v), 8, 64)
+ if err == nil && flags&unix.O_CLOEXEC != 0 {
+ return true
+ }
+ }
+ }
+ }
+ return false
+}
+
+func TestShimKernelSignalBeforePlacement(t *testing.T) {
+ m := shimKernelManager(t)
+ dir := shimKernelTarget(t, m, "sigplace")
+ setShimVar(t, &shimTestHold, 1)
+ for _, sig := range []syscall.Signal{syscall.SIGTERM, syscall.SIGKILL} {
+ orig := startCmd
+ held := make(chan bool, 1)
+ startCmd = func(cmd *exec.Cmd) error {
+ if err := orig(cmd); err != nil {
+ return err
+ }
+ // The spec fd is the first of the shim's three, after the
+ // tenant's ExtraFiles: 3 + (len(cmd.ExtraFiles) - 3).
+ specFD := len(cmd.ExtraFiles)
+ go func() {
+ held <- waitShimHeld(cmd.Process.Pid, specFD)
+ _ = cmd.Process.Signal(sig)
+ }()
+ return nil
+ }
+ marker := filepath.Join(t.TempDir(), "ran")
+ cmd := markerCmd(marker, "")
+ _, err := m.startViaShim(dir, cmd, nil)
+ startCmd = orig
+ if !<-held {
+ t.Fatalf("%v: the shim never reached its hold", sig)
+ }
+ if !isPlacementError(err) || !strings.Contains(err.Error(), "exited before placement") {
+ t.Fatalf("%v: err=%v, want a placement error, exited before placement", sig, err)
+ }
+ t.Logf("%v: %v", sig, err)
+ assertReaped(t, cmd)
+ assertNoMarker(t, marker)
+ assertNothingInFlight(t, m)
+ }
+}
+
+func TestShimKernelSignalBeforeExec(t *testing.T) {
+ m := shimKernelManager(t)
+ dir := shimKernelTarget(t, m, "sigexec")
+ setShimVar(t, &shimTestHold, 2)
+ marker := filepath.Join(t.TempDir(), "ran")
+ cmd := markerCmd(marker, "")
+ setShimVar(t, &shimAfterRelease, func() { _ = cmd.Process.Kill() })
+ _, err := m.startViaShim(dir, cmd, nil)
+ if !isPlacementError(err) || !strings.Contains(err.Error(), "exited before exec") {
+ t.Fatalf("err=%v, want a placement error, exited before exec", err)
+ }
+ assertReaped(t, cmd)
+ assertNoMarker(t, marker)
+ assertNothingInFlight(t, m)
+}
+
+// TestShimKernelExecBeforeAcknowledgement pins the rule that an exec that
+// happened is never reported as a failure before exec: the tenant runs, the
+// parent's deadline passes before it reads R, and the start is still
+// reported started.
+func TestShimKernelExecBeforeAcknowledgement(t *testing.T) {
+ m := shimKernelManager(t)
+ dir := shimKernelTarget(t, m, "lateack")
+ setShimVar(t, &shimDeadline, shimDeadline)
+ // The short deadline goes in once the shim is started, so it governs the
+ // release-to-exec read the test is about, and a loaded box's slow shim
+ // start cannot fail the placement read instead.
+ orig := startCmd
+ t.Cleanup(func() { startCmd = orig })
+ startCmd = func(cmd *exec.Cmd) error {
+ shimDeadline = 200 * time.Millisecond
+ return orig(cmd)
+ }
+ marker := filepath.Join(t.TempDir(), "ran")
+ cmd := markerCmd(marker, "")
+ setShimVar(t, &shimAfterRelease, func() {
+ for deadline := time.Now().Add(5 * time.Second); time.Now().Before(deadline); time.Sleep(time.Millisecond) {
+ if _, err := os.Stat(marker); err == nil {
+ break
+ }
+ }
+ time.Sleep(400 * time.Millisecond)
+ })
+ refused, err := m.startViaShim(dir, cmd, nil)
+ if refused || err != nil {
+ t.Fatalf("refused=%v err=%v, want started", refused, err)
+ }
+ if err := cmd.Wait(); err != nil {
+ t.Fatalf("Wait = %v", err)
+ }
+ if _, err := os.Stat(marker); err != nil {
+ t.Fatal("the tenant did not run")
+ }
+}
+
+func TestShimKernelDeadlineBeforePlacement(t *testing.T) {
+ m := shimKernelManager(t)
+ dir := shimKernelTarget(t, m, "deadline")
+ setShimVar(t, &shimTestHold, 1)
+ setShimVar(t, &shimDeadline, 300*time.Millisecond)
+ marker := filepath.Join(t.TempDir(), "ran")
+ cmd := markerCmd(marker, "")
+ _, err := m.startViaShim(dir, cmd, nil)
+ if !isPlacementError(err) || !strings.Contains(err.Error(), "no answer from the shim within 300ms") {
+ t.Fatalf("err=%v, want a placement error naming the deadline", err)
+ }
+ assertReaped(t, cmd)
+ assertNoMarker(t, marker)
+ assertNothingInFlight(t, m)
+}
+
+// TestShimKernelSurvivesDeletedExecutable: a roll replaces the carlos binary
+// on disk under a running agent; /proc/self/exe still execs the build the
+// agent is running.
+func TestShimKernelSurvivesDeletedExecutable(t *testing.T) {
+ shimKernelManager(t)
+ self, err := os.Executable()
+ if err != nil {
+ t.Fatal(err)
+ }
+ copyPath := filepath.Join(t.TempDir(), "copy.test")
+ src, err := os.Open(self)
+ if err != nil {
+ t.Fatal(err)
+ }
+ dst, err := os.OpenFile(copyPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o755)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := io.Copy(dst, src); err != nil {
+ t.Fatal(err)
+ }
+ src.Close()
+ if err := dst.Close(); err != nil {
+ t.Fatal(err)
+ }
+ child := exec.Command(copyPath, "-test.run=^TestShimKernelDeletedChild$", "-test.v", "-test.count=1")
+ child.Env = []string{
+ "CARLOS_SHIM_KERNEL=1", "CARLOS_SHIM_DELETED_CHILD=1", "CARLOS_GEN_KERNEL=1",
+ "TMPDIR=" + os.TempDir(), "PATH=" + os.Getenv("PATH"),
+ }
+ out, err := child.CombinedOutput()
+ t.Logf("child:\n%s", out)
+ if err != nil || !strings.Contains(string(out), "--- PASS: TestShimKernelDeletedChild") {
+ t.Fatalf("the child with a deleted executable failed: %v", err)
+ }
+}
+
+// TestShimKernelDeletedChild is TestShimKernelSurvivesDeletedExecutable's
+// child. Outside that run it does nothing.
+func TestShimKernelDeletedChild(t *testing.T) {
+ if os.Getenv("CARLOS_SHIM_DELETED_CHILD") != "1" {
+ return
+ }
+ if err := os.Remove(os.Args[0]); err != nil {
+ t.Fatal(err)
+ }
+ exe, err := os.Readlink("/proc/self/exe")
+ if err != nil || !strings.HasSuffix(exe, " (deleted)") {
+ t.Fatalf("/proc/self/exe = %q (%v), want a deleted file", exe, err)
+ }
+ t.Logf("running from %s", exe)
+ m := shimKernelManager(t)
+ dir := shimKernelTarget(t, m, "deleted")
+ cmd := exec.Command("/bin/sh", "-c", "cat /proc/self/cgroup")
+ var out bytes.Buffer
+ cmd.Stdout = &out
+ if refused, err := m.startViaShim(dir, cmd, nil); refused || err != nil {
+ t.Fatalf("refused=%v err=%v", refused, err)
+ }
+ if err := cmd.Wait(); err != nil {
+ t.Fatal(err)
+ }
+ if got, want := strings.TrimSpace(out.String()), "0::"+shimCgroupPath(dir); got != want {
+ t.Fatalf("tenant ran in %q, want %q", got, want)
+ }
+}
+
+// TestShimKernelOverhead is informational: it never fails on time.
+func TestShimKernelOverhead(t *testing.T) {
+ m := shimKernelManager(t)
+ dir := shimKernelTarget(t, m, "overhead")
+ const n = 20
+ median := func(viaShim bool) time.Duration {
+ var d []time.Duration
+ for range n {
+ cmd := exec.Command("/bin/true")
+ start := time.Now()
+ var err error
+ if viaShim {
+ _, err = m.startViaShim(dir, cmd, nil)
+ } else {
+ err = cmd.Start()
+ }
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := cmd.Wait(); err != nil {
+ t.Fatal(err)
+ }
+ d = append(d, time.Since(start))
+ }
+ slices.Sort(d)
+ return d[n/2]
+ }
+ shim := median(true)
+ direct := median(false)
+ t.Logf("start-to-exit of /bin/true, median of %d: shim %v, direct %v, overhead %v", n, shim, direct, shim-direct)
+}
diff --git a/internal/cgroup/shim_linux.go b/internal/cgroup/shim_linux.go
new file mode 100644
index 000000000..491bb7f3a
--- /dev/null
+++ b/internal/cgroup/shim_linux.go
@@ -0,0 +1,554 @@
+//go:build linux
+
+package cgroup
+
+import (
+ "bufio"
+ "bytes"
+ "encoding/binary"
+ "errors"
+ "fmt"
+ "io"
+ "math"
+ "os"
+ "os/exec"
+ "runtime"
+ "slices"
+ "strconv"
+ "strings"
+ "syscall"
+ "testing"
+ "time"
+
+ "golang.org/x/sys/unix"
+)
+
+// shimAvailable reports that RunShimIfRequested ran in this process, so
+// /proc/self/exe will serve ShimVerb.
+func shimAvailable() bool { return shimHooked.Load() }
+
+// startViaShim starts cmd so that its process is in dir before any tenant
+// code runs: the child starts as this binary (shimExe ShimVerb), writes its
+// own PID into dir's cgroup.procs, checks where it landed, answers placed,
+// and execs the tenant only once released.
+//
+// On a nil error the tenant is running in dir and cmd.Process is it.
+// refused means before said no: the shim was placed, never released, and
+// is reaped. Any other error means no tenant code ran and any shim process
+// is reaped: a *placementError is the placement's failure, anything else is
+// the command's own error, as exec.Cmd.Start would have returned it. A
+// failed start's cmd is spent and must not be reused.
+//
+// The classification rests on one rule: the shim writes R immediately
+// before execve, and never earlier. A start is reported failed only when
+// the status stream proves no R; a start that may have exec'd is reported
+// started, so its caller owns the process and nothing runs a tenant twice.
+func (m *Manager) startViaShim(dir string, cmd *exec.Cmd, before func() error) (refused bool, err error) {
+ if !shimAvailable() {
+ return false, &placementError{errors.New("the placement shim is not available in this process")}
+ }
+ if cmd.Err != nil {
+ return false, cmd.Err
+ }
+ fd, err := unix.Open(dir, unix.O_RDONLY|unix.O_DIRECTORY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)
+ if err != nil {
+ return false, &placementError{fmt.Errorf("open %s: %w", dir, err)}
+ }
+ dirFile := os.NewFile(uintptr(fd), dir)
+ var st unix.Statfs_t
+ if err := unix.Fstatfs(fd, &st); err != nil {
+ dirFile.Close()
+ return false, &placementError{fmt.Errorf("statfs %s: %w", dir, err)}
+ }
+ if st.Type != unix.CGROUP2_SUPER_MAGIC {
+ dirFile.Close()
+ return false, &placementError{fmt.Errorf("%s is not a cgroup2 directory", dir)}
+ }
+ // The shim checks /proc/self/cgroup against this, and the path below
+ // /sys/fs/cgroup against the fd, so dir must be named there.
+ rel, ok := strings.CutPrefix(dir, "/sys/fs/cgroup/")
+ if !ok || rel == "" {
+ dirFile.Close()
+ return false, &placementError{fmt.Errorf("%s is not a cgroup below /sys/fs/cgroup", dir)}
+ }
+ return m.startShimCore(dir, dirFile, "/"+rel, cmd, before)
+}
+
+// startShimCore is startViaShim after its checks: dirFile is dir opened,
+// and expected is dir as /proc/self/cgroup will show it. It closes dirFile.
+func (m *Manager) startShimCore(dir string, dirFile *os.File, expected string, cmd *exec.Cmd, before func() error) (refused bool, err error) {
+ // The tenant, captured before cmd is rewritten into the shim's start:
+ // exactly what exec.Cmd.Start would have passed.
+ path := cmd.Path
+ argv := cmd.Args
+ if len(argv) == 0 {
+ argv = []string{cmd.Path}
+ }
+ wd := cmd.Dir
+ keep := 3 + len(cmd.ExtraFiles)
+ spec, err := encodeShimSpec(shimSpec{
+ Path: path, Dir: wd, Argv: argv, Env: cmd.Environ(),
+ Expected: expected, KeepFDs: keep, TestHold: shimTestHold,
+ })
+ if err != nil {
+ // A NUL execve cannot carry: what execve answers for it.
+ dirFile.Close()
+ return false, &os.PathError{Op: "fork/exec", Path: path, Err: syscall.EINVAL}
+ }
+ if len(spec) > shimSpecMax {
+ dirFile.Close()
+ return false, &os.PathError{Op: "fork/exec", Path: path, Err: syscall.E2BIG}
+ }
+
+ specR, specW, err := os.Pipe()
+ if err != nil {
+ dirFile.Close()
+ return false, &placementError{fmt.Errorf("spec pipe: %w", err)}
+ }
+ statusR, statusW, err := os.Pipe()
+ if err != nil {
+ dirFile.Close()
+ specR.Close()
+ specW.Close()
+ return false, &placementError{fmt.Errorf("status pipe: %w", err)}
+ }
+ // Every return closes specW and statusR exactly once; specW may go
+ // early, as the release or the refusal.
+ specWOpen := true
+ closeSpecW := func() {
+ if specWOpen {
+ specWOpen = false
+ specW.Close()
+ }
+ }
+ defer closeSpecW()
+ defer statusR.Close()
+
+ // The shim's own start: no tenant environment and no working directory
+ // (the shim changes to wd itself, after placement, so a bad wd is the
+ // command's error, never the placement's). Stdin, Stdout, Stderr,
+ // SysProcAttr, Cancel and WaitDelay stay the caller's.
+ cmd.Path = shimExe
+ cmd.Args = []string{"carlos-cgexec", ShimVerb, strconv.Itoa(keep), strconv.Itoa(keep + 1), strconv.Itoa(keep + 2)}
+ cmd.Env = []string{}
+ cmd.Dir = ""
+ cmd.ExtraFiles = append(slices.Clip(cmd.ExtraFiles), specR, dirFile, statusW)
+
+ // One deadline covers the spec and the wait for placement.
+ deadline := time.Now().Add(shimDeadline)
+ done := m.beginPlacement(dir)
+ startErr := startCmd(cmd)
+ specR.Close()
+ dirFile.Close()
+ statusW.Close()
+ if startErr != nil {
+ // The carlos binary itself did not start; the tenant was not part
+ // of this start.
+ done()
+ return false, &placementError{fmt.Errorf("start the shim: %w", startErr)}
+ }
+
+ msg := make([]byte, 8+len(spec))
+ binary.BigEndian.PutUint64(msg, uint64(len(spec)))
+ copy(msg[8:], spec)
+ _ = specW.SetWriteDeadline(deadline)
+ if _, err := specW.Write(msg); err != nil {
+ reap(cmd)
+ done()
+ if errors.Is(err, os.ErrDeadlineExceeded) {
+ return false, &placementError{fmt.Errorf("no answer from the shim within %v", shimDeadline)}
+ }
+ return false, &placementError{fmt.Errorf("the shim exited before placement (spec: %v)", err)}
+ }
+
+ _ = statusR.SetReadDeadline(deadline)
+ br := bufio.NewReader(statusR)
+ line, rerr := br.ReadBytes('\n')
+ placed, perr := classifyPlacement(line, rerr != nil && !errors.Is(rerr, io.EOF))
+ if !placed {
+ // P, EOF or the deadline: no release byte was sent, so the shim
+ // cannot have exec'd. reap's SIGKILL ends one still running.
+ reap(cmd)
+ done()
+ return false, perr
+ }
+ // The process is in dir now, visible to every reader of the cgroup: the
+ // registry covers only fork to placement, and no caller code runs
+ // inside it.
+ done()
+
+ // The caller's last check, after placement and immediately before exec
+ // (internal/activator/genbackend.go). A refusal closes specW without the
+ // release byte: the shim reads EOF and exits without exec.
+ if before != nil {
+ if err := before(); err != nil {
+ closeSpecW()
+ reap(cmd)
+ return true, err
+ }
+ }
+
+ // Release. A failed write leaves the shim to read EOF; the status read
+ // below then proves no R.
+ _ = specW.SetWriteDeadline(time.Time{})
+ _, _ = specW.Write([]byte{'X'})
+ closeSpecW()
+ _ = statusR.SetReadDeadline(time.Now().Add(shimDeadline))
+ if shimAfterRelease != nil {
+ shimAfterRelease()
+ }
+ rest, rerr := io.ReadAll(br)
+ killed := false
+ if rerr != nil {
+ // The deadline passed before EOF. With R read, an execve is still
+ // loading: started, and nothing is killed. An E line is the shim's
+ // answer. Otherwise kill the shim, which was not released into an
+ // exec it has announced, and read what it wrote before it died.
+ started, err := classifyExec(path, wd, rest, false)
+ if started {
+ return false, nil
+ }
+ if !isPlacementError(err) {
+ reap(cmd)
+ return false, err
+ }
+ _ = cmd.Process.Kill()
+ _ = statusR.SetReadDeadline(time.Time{})
+ more, _ := io.ReadAll(br)
+ rest = append(rest, more...)
+ killed = true
+ }
+ started, err := classifyExec(path, wd, rest, killed)
+ if started {
+ return false, nil
+ }
+ reap(cmd)
+ return false, err
+}
+
+// reap ends a failed start's shim: kill, then wait, which reaps the PID and
+// is bounded because no tenant ran to hold the child's pipes open. exec.Cmd's
+// copying goroutines and pipes are released in the background, so the
+// caller's I/O never holds this call.
+func reap(cmd *exec.Cmd) {
+ _ = cmd.Process.Kill()
+ _, _ = cmd.Process.Wait()
+ go func() { _ = cmd.Wait() }()
+}
+
+// classifyPlacement reads the stream up to placement (parent step 7).
+// placed is true only for "A". timedOut is the read deadline having passed.
+func classifyPlacement(b []byte, timedOut bool) (placed bool, err error) {
+ if i := bytes.IndexByte(b, '\n'); i >= 0 {
+ line := string(b[:i])
+ switch {
+ case line == "A":
+ return true, nil
+ case strings.HasPrefix(line, "P "):
+ return false, &placementError{fmt.Errorf("the shim could not place the process: %s", line[2:])}
+ default:
+ return false, &placementError{fmt.Errorf("the shim answered %q before placement", line)}
+ }
+ }
+ if timedOut {
+ return false, &placementError{fmt.Errorf("no answer from the shim within %v", shimDeadline)}
+ }
+ return false, &placementError{errors.New("the shim exited before placement")}
+}
+
+// classifyExec reads the stream after the release byte (parent step 9).
+// killed means the deadline passed with no R, the shim was SIGKILLed, and
+// b is everything read until EOF after that.
+//
+// R, and no "E exec" after it, is started: the tenant may have run. An
+// "E chdir" before R, or an "E exec" after it, is the command's own error,
+// as exec.Cmd.Start reports it. No R is a placement error.
+func classifyExec(path, wd string, b []byte, killed bool) (started bool, err error) {
+ sawR := false
+ for _, line := range strings.Split(string(b), "\n") {
+ if line == "R" {
+ sawR = true
+ continue
+ }
+ op, n, ok := parseShimErr(line)
+ switch {
+ case !ok:
+ case !sawR && op == "chdir":
+ return false, &os.PathError{Op: "chdir", Path: wd, Err: syscall.Errno(n)}
+ case sawR && op == "exec":
+ return false, &os.PathError{Op: "fork/exec", Path: path, Err: syscall.Errno(n)}
+ }
+ }
+ if sawR {
+ return true, nil
+ }
+ if killed {
+ return false, &placementError{fmt.Errorf("the shim did not exec within %v", shimDeadline)}
+ }
+ return false, &placementError{errors.New("the shim exited before exec")}
+}
+
+// parseShimErr parses "E <op> <errno>".
+func parseShimErr(line string) (op string, errno int, ok bool) {
+ f := strings.Fields(line)
+ if len(f) != 3 || f[0] != "E" {
+ return "", 0, false
+ }
+ n, err := strconv.Atoi(f[2])
+ if err != nil || n <= 0 {
+ return "", 0, false
+ }
+ return f[1], n, true
+}
+
+// runShim is the shim process: args are the spec, directory and status
+// fds, which are keep, keep+1 and keep+2. It writes nothing to fds 0 to 2.
+// A failure before placement is confirmed writes "P <step>: <error>" and
+// returns 125; tenant code has not run.
+func runShim(args []string) int {
+ if len(args) != 3 {
+ return 125
+ }
+ var fds [3]int
+ for i, a := range args {
+ n, err := strconv.Atoi(a)
+ if err != nil {
+ return 125
+ }
+ fds[i] = n
+ }
+ keep, specFD, dirFD, statusFD := fds[0], fds[0], fds[1], fds[2]
+ if keep < 3 || dirFD != keep+1 || statusFD != keep+2 {
+ return 125
+ }
+ // First, so every later path, the exec included, closes it.
+ if _, err := unix.FcntlInt(uintptr(statusFD), unix.F_SETFD, unix.FD_CLOEXEC); err != nil {
+ return 125
+ }
+ fail := func(step string, err error) int {
+ shimWrite(statusFD, "P "+step+": "+err.Error()+"\n")
+ return 125
+ }
+ // The pipes may carry the parent's O_NONBLOCK; the shim's reads and
+ // writes block.
+ for _, fd := range []int{specFD, statusFD} {
+ if err := unix.SetNonblock(fd, false); err != nil {
+ return fail("fds", err)
+ }
+ }
+ spec, err := readShimSpec(specFD)
+ if err != nil {
+ return fail("spec", err)
+ }
+ if spec.KeepFDs != keep {
+ return fail("spec", fmt.Errorf("the spec keeps %d fds, the shim was given %d", spec.KeepFDs, keep))
+ }
+ // Everything from keep up closes at exec: the shim's three fds and any
+ // the agent leaked without close-on-exec.
+ if err := cloexecFrom(keep); err != nil {
+ return fail("cloexec", err)
+ }
+ shimHold(spec.TestHold, 1)
+ var st unix.Statfs_t
+ if err := unix.Fstatfs(dirFD, &st); err != nil {
+ return fail("statfs", err)
+ }
+ if st.Type != unix.CGROUP2_SUPER_MAGIC {
+ return fail("statfs", errors.New("not a cgroup2 directory"))
+ }
+ // Moves the whole process, every Go runtime thread, while it is still
+ // carlos code.
+ if err := joinCgroup(dirFD); err != nil {
+ return fail("procs", err)
+ }
+ if err := verifyCgroup(dirFD, spec.Expected); err != nil {
+ return fail("verify", err)
+ }
+ unix.Close(dirFD)
+ if !shimWrite(statusFD, "A\n") {
+ return 125
+ }
+ // The release byte. EOF, an error or another byte: the parent refused
+ // or is gone.
+ var rel [1]byte
+ if n, err := unix.Read(specFD, rel[:]); n != 1 || err != nil || rel[0] != 'X' {
+ return 125
+ }
+ shimHold(spec.TestHold, 2)
+ if spec.Dir != "" {
+ if err := os.Chdir(spec.Dir); err != nil {
+ shimWrite(statusFD, fmt.Sprintf("E chdir %d\n", errnoOf(err)))
+ return 126
+ }
+ }
+ // The exec state a direct start under systemd would have: an empty
+ // signal mask on the thread that execs. execve resets every handled
+ // signal to default and keeps ignored ones ignored, and syscall.Exec
+ // restores the RLIMIT_NOFILE this process started with.
+ runtime.LockOSThread()
+ if err := unix.PthreadSigmask(unix.SIG_SETMASK, &unix.Sigset_t{}, nil); err != nil {
+ return 125
+ }
+ // R immediately before execve, and never earlier: without it the parent
+ // may report the start failed, so no exec may follow a failed write.
+ if !shimWrite(statusFD, "R\n") {
+ return 125
+ }
+ err = syscall.Exec(spec.Path, spec.Argv, spec.Env)
+ shimWrite(statusFD, fmt.Sprintf("E exec %d\n", errnoOf(err)))
+ return 126
+}
+
+// runProbe prints this process's cgroup as /proc/self/cgroup shows it.
+func runProbe() int {
+ raw, err := os.ReadFile("/proc/self/cgroup")
+ if err != nil {
+ return 1
+ }
+ rel, ok := ParseSelfCgroup(string(raw))
+ if !ok {
+ return 1
+ }
+ if _, err := os.Stdout.WriteString("/" + rel + "\n"); err != nil {
+ return 1
+ }
+ return 0
+}
+
+// readShimSpec reads the 8-byte big-endian length and then the spec.
+func readShimSpec(fd int) (shimSpec, error) {
+ var hdr [8]byte
+ if err := readFull(fd, hdr[:]); err != nil {
+ return shimSpec{}, err
+ }
+ n := binary.BigEndian.Uint64(hdr[:])
+ if n > shimSpecMax {
+ return shimSpec{}, fmt.Errorf("spec of %d bytes is over %d", n, shimSpecMax)
+ }
+ b := make([]byte, n)
+ if err := readFull(fd, b); err != nil {
+ return shimSpec{}, err
+ }
+ return decodeShimSpec(b)
+}
+
+func readFull(fd int, b []byte) error {
+ for len(b) > 0 {
+ n, err := unix.Read(fd, b)
+ if err == unix.EINTR {
+ continue
+ }
+ if err != nil {
+ return err
+ }
+ if n == 0 {
+ return io.ErrUnexpectedEOF
+ }
+ b = b[n:]
+ }
+ return nil
+}
+
+func shimWrite(fd int, s string) bool {
+ b := []byte(s)
+ for len(b) > 0 {
+ n, err := unix.Write(fd, b)
+ if err == unix.EINTR {
+ continue
+ }
+ if err != nil {
+ return false
+ }
+ b = b[n:]
+ }
+ return true
+}
+
+// cloexecFrom marks every fd >= keep close-on-exec. Without close_range
+// (ENOSYS on an old kernel, or a filter's EPERM or EINVAL) it walks
+// /proc/self/fd.
+func cloexecFrom(keep int) error {
+ err := unix.CloseRange(uint(keep), math.MaxUint32, unix.CLOSE_RANGE_CLOEXEC)
+ if err == nil {
+ return nil
+ }
+ if err != unix.ENOSYS && err != unix.EPERM && err != unix.EINVAL {
+ return err
+ }
+ entries, err := os.ReadDir("/proc/self/fd")
+ if err != nil {
+ return err
+ }
+ for _, e := range entries {
+ n, err := strconv.Atoi(e.Name())
+ if err != nil || n < keep {
+ continue
+ }
+ // EBADF is the listing's own fd, closed since.
+ if _, err := unix.FcntlInt(uintptr(n), unix.F_SETFD, unix.FD_CLOEXEC); err != nil && err != unix.EBADF {
+ return err
+ }
+ }
+ return nil
+}
+
+// joinCgroup writes this process's PID into the cgroup.procs of the
+// directory dirFD names.
+func joinCgroup(dirFD int) error {
+ fd, err := unix.Openat(dirFD, "cgroup.procs", unix.O_WRONLY|unix.O_NOFOLLOW|unix.O_CLOEXEC, 0)
+ if err != nil {
+ return fmt.Errorf("open cgroup.procs: %w", err)
+ }
+ defer unix.Close(fd)
+ if _, err := unix.Write(fd, []byte(strconv.Itoa(os.Getpid()))); err != nil {
+ return fmt.Errorf("write cgroup.procs: %w", err)
+ }
+ return nil
+}
+
+// verifyCgroup checks that the kernel reports this process in expected, and
+// that expected is the directory dirFD names (same device and inode).
+func verifyCgroup(dirFD int, expected string) error {
+ raw, err := os.ReadFile("/proc/self/cgroup")
+ if err != nil {
+ return err
+ }
+ rel, ok := ParseSelfCgroup(string(raw))
+ if !ok {
+ return errors.New("no cgroup v2 path in /proc/self/cgroup")
+ }
+ if "/"+rel != expected {
+ return fmt.Errorf("landed in /%s, want %s", rel, expected)
+ }
+ var fst, lst unix.Stat_t
+ if err := unix.Fstat(dirFD, &fst); err != nil {
+ return err
+ }
+ if err := unix.Lstat("/sys/fs/cgroup"+expected, &lst); err != nil {
+ return err
+ }
+ if fst.Dev != lst.Dev || fst.Ino != lst.Ino {
+ return fmt.Errorf("/sys/fs/cgroup%s is not the directory the parent opened", expected)
+ }
+ return nil
+}
+
+// shimHold sleeps for good at the point at, when a test binary's spec asks
+// for it, until a signal ends the process. Default dispositions: no
+// signal.Notify, so SIGTERM or SIGKILL kills it there.
+func shimHold(hold, at int) {
+ if hold != at || !testing.Testing() {
+ return
+ }
+ for {
+ time.Sleep(time.Hour)
+ }
+}
+
+func errnoOf(err error) int {
+ var e syscall.Errno
+ if errors.As(err, &e) {
+ return int(e)
+ }
+ return int(syscall.EINVAL)
+}
diff --git a/internal/cgroup/shim_linux_test.go b/internal/cgroup/shim_linux_test.go
new file mode 100644
index 000000000..3bfedeabd
--- /dev/null
+++ b/internal/cgroup/shim_linux_test.go
@@ -0,0 +1,236 @@
+//go:build linux
+
+package cgroup
+
+import (
+ "errors"
+ "fmt"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "syscall"
+ "testing"
+)
+
+// These run without a delegated cgroup: a temp directory is not cgroupfs,
+// so they exercise the failure paths. The kernel file covers the rest.
+
+// TestShimRefusesANonCgroupDirectory: the parent's own check (step 2)
+// refuses before anything starts, as a placement error.
+func TestShimRefusesANonCgroupDirectory(t *testing.T) {
+ m := &Manager{root: t.TempDir(), logf: t.Logf}
+ marker := filepath.Join(t.TempDir(), "ran")
+ cmd := exec.Command("/bin/sh", "-c", "touch "+marker)
+ refused, err := m.startViaShim(m.root, cmd, nil)
+ if refused || !isPlacementError(err) {
+ t.Fatalf("refused=%v err=%v, want a placement error", refused, err)
+ }
+ if cmd.Process != nil {
+ t.Fatal("a process was started")
+ }
+ if _, err := os.Stat(marker); err == nil {
+ t.Fatal("tenant code ran")
+ }
+}
+
+// TestShimRefusesAPathOutsideCgroupfs: a cgroup2 directory is not enough;
+// the path must name it under /sys/fs/cgroup/, or the shim could not check
+// where it landed.
+func TestShimRefusesAPathOutsideCgroupfs(t *testing.T) {
+ if _, err := os.Stat("/sys/fs/cgroup/cgroup.controllers"); err != nil {
+ t.Skip("no cgroup2 mount at /sys/fs/cgroup")
+ }
+ m := &Manager{root: "/sys/fs/cgroup", logf: t.Logf}
+ cmd := exec.Command("/bin/true")
+ _, err := m.startViaShim("/sys/fs/cgroup", cmd, nil)
+ if !isPlacementError(err) || cmd.Process != nil {
+ t.Fatalf("err=%v process=%v, want a placement error and no process", err, cmd.Process)
+ }
+}
+
+// TestShimUnavailableWithoutTheHook: a process whose main never ran the
+// hook never starts a shim.
+func TestShimUnavailableWithoutTheHook(t *testing.T) {
+ if !shimHooked.Load() {
+ t.Fatal("TestMain did not run the hook")
+ }
+ shimHooked.Store(false)
+ t.Cleanup(func() { shimHooked.Store(true) })
+ m := &Manager{root: t.TempDir(), logf: t.Logf}
+ cmd := exec.Command("/bin/true")
+ refused, err := m.startViaShim(m.root, cmd, nil)
+ if refused || !isPlacementError(err) {
+ t.Fatalf("refused=%v err=%v, want a placement error", refused, err)
+ }
+ if cmd.Process != nil {
+ t.Fatal("a process was started")
+ }
+}
+
+// TestShimLookPathErrorIsTheCommands: a command whose LookPath failed
+// answers that error, as exec.Cmd.Start would, and starts nothing.
+func TestShimLookPathErrorIsTheCommands(t *testing.T) {
+ m := &Manager{root: t.TempDir(), logf: t.Logf}
+ cmd := exec.Command("carlos-no-such-command-anywhere")
+ if cmd.Err == nil {
+ t.Skip("LookPath found the command")
+ }
+ _, err := m.startViaShim(m.root, cmd, nil)
+ if err != cmd.Err || isPlacementError(err) || cmd.Process != nil {
+ t.Fatalf("err=%v, want the LookPath error %v and no process", err, cmd.Err)
+ }
+}
+
+// TestShimPlacementFailureIsReported runs a real shim (this test binary,
+// through TestMain's hook) against a temp directory, calling the core with
+// a directory fd the prologue would have refused. The shim's own fstatfs
+// check (shim step 4) answers "P".
+func TestShimPlacementFailureIsReported(t *testing.T) {
+ m := &Manager{root: t.TempDir(), logf: t.Logf}
+ f, err := os.Open(m.root)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+ marker := filepath.Join(t.TempDir(), "ran")
+ cmd := exec.Command("/bin/sh", "-c", "touch "+marker)
+ _, err = m.startShimCore(m.root, f, "/nowhere", cmd, nil)
+ if !isPlacementError(err) {
+ t.Fatalf("err=%v, want a placement error", err)
+ }
+ if !strings.Contains(err.Error(), "statfs") {
+ t.Fatalf("err=%v, want the shim's statfs refusal", err)
+ }
+ if _, err := os.Stat(fmt.Sprintf("/proc/%d", cmd.Process.Pid)); err == nil {
+ t.Fatal("the shim was not reaped")
+ }
+ if _, err := os.Stat(marker); err == nil {
+ t.Fatal("tenant code ran")
+ }
+ if m.placementsUnder(m.root) != 0 {
+ t.Fatal("the placement is still registered")
+ }
+}
+
+// TestShimBeforeRefusalRegistersNothing: before is asked only once the
+// process is placed, so a start that cannot be placed never asks it, starts
+// nothing of the tenant, and leaves nothing registered. A refusal after a
+// real placement is TestShimKernelBeforeRunsAfterPlacement.
+func TestShimBeforeRefusalRegistersNothing(t *testing.T) {
+ m := &Manager{root: t.TempDir(), logf: t.Logf}
+ asked := 0
+ before := func() error { asked++; return errors.New("not yours any more") }
+ marker := filepath.Join(t.TempDir(), "ran")
+
+ cmd := exec.Command("/bin/sh", "-c", "touch "+marker)
+ refused, err := m.startViaShim(m.root, cmd, before)
+ if refused || !isPlacementError(err) || cmd.Process != nil {
+ t.Fatalf("prologue: refused=%v err=%v process=%v", refused, err, cmd.Process)
+ }
+
+ f, err := os.Open(m.root)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer f.Close()
+ cmd = exec.Command("/bin/sh", "-c", "touch "+marker)
+ refused, err = m.startShimCore(m.root, f, "/nowhere", cmd, before)
+ if refused || !isPlacementError(err) {
+ t.Fatalf("core: refused=%v err=%v, want a placement error", refused, err)
+ }
+ if asked != 0 {
+ t.Fatalf("before was asked %d times for a process that was never placed", asked)
+ }
+ if _, err := os.Stat(marker); err == nil {
+ t.Fatal("tenant code ran")
+ }
+ if m.placementsUnder(m.root) != 0 {
+ t.Fatal("the placement is still registered")
+ }
+}
+
+func TestShimStatusClassification(t *testing.T) {
+ for _, c := range []struct {
+ in string
+ timedOut bool
+ placed bool
+ }{
+ {"A\n", false, true},
+ {"P procs: permission denied\n", false, false},
+ {"", false, false}, // died before placement
+ {"", true, false}, // deadline before placement
+ {"garbage\n", false, false},
+ {"A", false, false}, // no newline: not the shim's answer
+ } {
+ placed, err := classifyPlacement([]byte(c.in), c.timedOut)
+ if placed != c.placed {
+ t.Fatalf("classifyPlacement(%q, %v) placed=%v, want %v", c.in, c.timedOut, placed, c.placed)
+ }
+ if !placed && !isPlacementError(err) {
+ t.Fatalf("classifyPlacement(%q, %v) err=%v, want a placement error", c.in, c.timedOut, err)
+ }
+ if placed && err != nil {
+ t.Fatalf("classifyPlacement(%q, %v) err=%v", c.in, c.timedOut, err)
+ }
+ }
+ if _, err := classifyPlacement([]byte("P procs: permission denied\n"), false); !strings.Contains(err.Error(), "procs: permission denied") {
+ t.Fatalf("the shim's reason is lost: %v", err)
+ }
+ if _, err := classifyPlacement(nil, true); !strings.Contains(err.Error(), "no answer") {
+ t.Fatalf("the deadline is not named: %v", err)
+ }
+ if _, err := classifyPlacement(nil, false); !strings.Contains(err.Error(), "exited before placement") {
+ t.Fatalf("the early exit is not named: %v", err)
+ }
+
+ for _, c := range []struct {
+ in string
+ killed bool
+ started bool
+ op string // "" for a placement error, else the *os.PathError's Op
+ errno syscall.Errno
+ }{
+ {"R\n", false, true, "", 0},
+ {"R\n", true, true, "", 0}, // R then killed: may have exec'd, so started
+ {"", false, false, "", 0}, // died after release, before R: placement error
+ {"", true, false, "", 0}, // killed before R: placement error
+ {"E chdir 2\n", false, false, "chdir", syscall.ENOENT},
+ {"R\nE exec 2\n", false, false, "fork/exec", syscall.ENOENT},
+ {"R\nE exec 7\n", false, false, "fork/exec", syscall.E2BIG},
+ {"garbage\n", false, false, "", 0},
+ {"R\ngarbage\n", false, true, "", 0}, // R and no E after it: started
+ {"R", true, true, "", 0}, // R without its newline still counts
+ } {
+ started, err := classifyExec("/bin/app", "/wd", []byte(c.in), c.killed)
+ if started != c.started {
+ t.Fatalf("classifyExec(%q, %v) started=%v, want %v", c.in, c.killed, started, c.started)
+ }
+ if started {
+ if err != nil {
+ t.Fatalf("classifyExec(%q, %v) err=%v", c.in, c.killed, err)
+ }
+ continue
+ }
+ if c.op == "" {
+ if !isPlacementError(err) {
+ t.Fatalf("classifyExec(%q, %v) err=%v, want a placement error", c.in, c.killed, err)
+ }
+ continue
+ }
+ var pe *os.PathError
+ if !errors.As(err, &pe) || isPlacementError(err) {
+ t.Fatalf("classifyExec(%q, %v) err=%v, want a *os.PathError", c.in, c.killed, err)
+ }
+ wantPath := "/bin/app"
+ if c.op == "chdir" {
+ wantPath = "/wd"
+ }
+ if pe.Op != c.op || pe.Path != wantPath || pe.Err != c.errno {
+ t.Fatalf("classifyExec(%q, %v) = %#v, want Op %q Path %q Err %v", c.in, c.killed, pe, c.op, wantPath, c.errno)
+ }
+ }
+ if _, err := classifyExec("/bin/app", "/wd", nil, false); !strings.Contains(err.Error(), "exited before exec") {
+ t.Fatalf("the early exit is not named: %v", err)
+ }
+}
diff --git a/internal/cgroup/shim_other.go b/internal/cgroup/shim_other.go
new file mode 100644
index 000000000..a1842c134
--- /dev/null
+++ b/internal/cgroup/shim_other.go
@@ -0,0 +1,20 @@
+//go:build !linux
+
+package cgroup
+
+import (
+ "errors"
+ "os/exec"
+)
+
+// The placement shim needs cgroup v2, so it is linux only.
+
+func shimAvailable() bool { return false }
+
+func runShim([]string) int { return 127 }
+
+func runProbe() int { return 127 }
+
+func (m *Manager) startViaShim(dir string, cmd *exec.Cmd, before func() error) (refused bool, err error) {
+ return false, &placementError{errors.New("placement before exec needs linux")}
+}
diff --git a/internal/cgroup/shim_test.go b/internal/cgroup/shim_test.go
new file mode 100644
index 000000000..191bd64c9
--- /dev/null
+++ b/internal/cgroup/shim_test.go
@@ -0,0 +1,74 @@
+package cgroup
+
+import (
+ "reflect"
+ "strings"
+ "testing"
+)
+
+func TestShimSpecRoundTripsExactBytes(t *testing.T) {
+ in := shimSpec{
+ Path: "/run/carlos/exe/app",
+ Dir: "/data/app dir",
+ Argv: []string{"app", "", "two words", "line\nbreak", "--db", "/data/x.db"},
+ Env: []string{"A=1", "EMPTY=", "EQ=a=b", "BAD=\xff\xfe", "BIG=" + strings.Repeat("x", 1<<20)},
+ Expected: "/system.slice/carlos-edge.service/carlos-exec@a.example/g3",
+ KeepFDs: 5,
+ TestHold: 2,
+ }
+ b, err := encodeShimSpec(in)
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := decodeShimSpec(b)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !reflect.DeepEqual(in, out) {
+ t.Fatalf("round trip changed the spec")
+ }
+}
+
+// An empty argv and env round-trip as empty, not as one empty string.
+func TestShimSpecRoundTripsEmptyLists(t *testing.T) {
+ in := shimSpec{Path: "/x", Argv: []string{}, Env: []string{}, Expected: "/g", KeepFDs: 3}
+ b, err := encodeShimSpec(in)
+ if err != nil {
+ t.Fatal(err)
+ }
+ out, err := decodeShimSpec(b)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(out.Argv) != 0 || len(out.Env) != 0 || out.Path != "/x" || out.Expected != "/g" || out.KeepFDs != 3 {
+ t.Fatalf("got %+v", out)
+ }
+}
+
+func TestShimSpecRefusesNUL(t *testing.T) {
+ if _, err := encodeShimSpec(shimSpec{Path: "/x", Argv: []string{"a\x00b"}, Expected: "/g", KeepFDs: 3}); err == nil {
+ t.Fatal("a NUL inside argv was encoded")
+ }
+ if _, err := encodeShimSpec(shimSpec{Path: "/x", Env: []string{"A=\x00"}, Expected: "/g", KeepFDs: 3}); err == nil {
+ t.Fatal("a NUL inside env was encoded")
+ }
+ if _, err := encodeShimSpec(shimSpec{Path: "/x\x00y", Expected: "/g", KeepFDs: 3}); err == nil {
+ t.Fatal("a NUL inside the path was encoded")
+ }
+}
+
+func TestShimSpecRefusesTruncatedAndTrailing(t *testing.T) {
+ b, _ := encodeShimSpec(shimSpec{Path: "/x", Argv: []string{"x"}, Env: []string{"A=1"}, Expected: "/g", KeepFDs: 3})
+ for _, bad := range [][]byte{b[:len(b)-1], append(append([]byte{}, b...), 'z', 0), []byte("carlos-cgexec-0\x00")} {
+ if _, err := decodeShimSpec(bad); err == nil {
+ t.Fatalf("decoded a bad spec %q", bad)
+ }
+ }
+}
+
+func TestShimSpecRefusesHugeCounts(t *testing.T) {
+ bad := []byte("carlos-cgexec-1\x00/x\x00\x00/g\x003\x000\x001048577\x00")
+ if _, err := decodeShimSpec(bad); err == nil {
+ t.Fatal("decoded an argv count above 1<<20")
+ }
+}
diff --git a/main.go b/main.go
index 2ea17257f..82add2feb 100644
--- a/main.go
+++ b/main.go
@@ -9,6 +9,7 @@ import (
"os"
"strings"
+ "github.com/carlosframework/platform/internal/cgroup"
"github.com/carlosframework/platform/internal/edge"
"github.com/carlosframework/platform/internal/host"
"github.com/carlosframework/platform/internal/instanceexec"
@@ -18,6 +19,10 @@ import (
)
func main() {
+ // The placement shim (internal/cgroup, #272) must run before anything
+ // else touches the environment or the logger: a shim process carries
+ // none of the tenant's settings and exits from inside this call.
+ cgroup.RunShimIfRequested()
rootaws.ScrubEnvironment()
// Failure is reported through log.Fatal everywhere in this binary, so
// the logger is where the "this went wrong" colour goes — once, rather