amadan

carlos / platform Public

cgroup: spawn into per-host cgroups — clone3 first, latched PID-move fallback

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Paul Campbell 8cbfd0e07e295292f457a381e93464deffd7fb42 parent 7c59aac
5 files changed, +275 −0
  • internal/cgroup/cgroup.go +86 −0
  • internal/cgroup/cgroup_linux.go +14 −0
  • internal/cgroup/cgroup_linux_test.go +81 −0
  • internal/cgroup/cgroup_other.go +9 −0
  • internal/cgroup/cgroup_test.go +85 −0
diff --git a/internal/cgroup/cgroup.go b/internal/cgroup/cgroup.go
index b7c27b529..e0b5fccb5 100644
--- a/internal/cgroup/cgroup.go
+++ b/internal/cgroup/cgroup.go
@@ -15,6 +15,7 @@ package cgroup
import (
"fmt"
"os"
+ "os/exec"
"path/filepath"
"strconv"
"strings"
@@ -162,3 +163,88 @@ func (m *Manager) sweepStale() {
}
}
}
+
+// startCmd is Start's spawn seam — a package var so the clone3 refusal
+// path is testable without a seccomp filter in the loop.
+var startCmd = func(cmd *exec.Cmd) error { return cmd.Start() }
+
+// hostDir is the per-host cgroup: instance, sidecar and seed all share it —
+// a seed's first-boot cost and a sidecar's load ARE that instance's work.
+func (m *Manager) hostDir(host string) string { return filepath.Join(m.root, ExecName(host)) }
+
+// Start runs build() and starts the result inside host's cgroup. Nil-safe:
+// with no manager (or any accounting failure) the child starts exactly as
+// today — a spawn must never fail because accounting failed. build must
+// return a fully configured, unstarted Cmd; it may be called twice, because
+// a Cmd cannot be reused after a failed Start. On a non-nil error the
+// returned Cmd may be nil.
+//
+// Primary mechanism: clone3 with CLONE_INTO_CGROUP (SysProcAttr.UseCgroupFD)
+// — the child never spends a cycle in the wrong cgroup. If that start
+// fails, the cgroup fd may or may not be why (a seccomp filter older than
+// clone3 answers EPERM; so does a missing binary): retry WITHOUT the fd,
+// and only if the retry succeeds — proving the cgroup was the problem —
+// latch the fallback and move the child by PID. The window between start
+// and move charges the child's first moments to the agent: mis-accounting
+// in the safe direction.
+func (m *Manager) Start(host string, build func() *exec.Cmd) (*exec.Cmd, error) {
+ if m == nil {
+ cmd := build()
+ return cmd, startCmd(cmd)
+ }
+ dir := m.hostDir(host)
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ m.logf("cgroup: %s: create cgroup: %v — spawning unaccounted", host, err)
+ cmd := build()
+ return cmd, startCmd(cmd)
+ }
+ if intoCgroupSupported && !m.fallback.Load() {
+ f, err := os.Open(dir)
+ if err != nil {
+ m.logf("cgroup: %s: open cgroup dir: %v — using PID-move fallback for this spawn", host, err)
+ return m.startAndMove(host, build)
+ }
+ cmd := build()
+ cmd.SysProcAttr = sysProcAttrInto(int(f.Fd()))
+ err = startCmd(cmd)
+ f.Close()
+ if err == nil {
+ return cmd, nil
+ }
+ firstErr := err
+ retried, retryErr := m.startAndMove(host, build)
+ if retryErr != nil {
+ return nil, retryErr // failed both ways: not the cgroup's fault
+ }
+ m.fallback.Store(true)
+ m.logf("cgroup: clone3-into-cgroup refused (%v) — latching PID-move fallback for the life of this process", firstErr)
+ return retried, nil
+ }
+ return m.startAndMove(host, build)
+}
+
+// startAndMove is the fallback spawn: plain start, then write the child's
+// PID into the host cgroup. A failed move is logged and tolerated — the
+// child runs (unaccounted), which beats not running.
+func (m *Manager) startAndMove(host string, build func() *exec.Cmd) (*exec.Cmd, error) {
+ cmd := build()
+ if err := startCmd(cmd); err != nil {
+ return nil, err
+ }
+ procs := filepath.Join(m.hostDir(host), "cgroup.procs")
+ if err := os.WriteFile(procs, []byte(strconv.Itoa(cmd.Process.Pid)), 0o644); err != nil {
+ m.logf("cgroup: %s: move pid %d: %v — child stays unaccounted", host, cmd.Process.Pid, err)
+ }
+ return cmd, nil
+}
+
+// CleanupHost removes host's per-host cgroup dir after its children exit.
+// Nil-safe and rmdir-tolerant: while a sibling (instance vs sidecar) is
+// still alive the dir is non-empty and the remove fails — the LAST exiter's
+// call succeeds, and the next agent start's sweep catches anything missed.
+func (m *Manager) CleanupHost(host string) {
+ if m == nil {
+ return
+ }
+ _ = os.Remove(m.hostDir(host))
+}
diff --git a/internal/cgroup/cgroup_linux.go b/internal/cgroup/cgroup_linux.go
new file mode 100644
index 000000000..80615ed85
--- /dev/null
+++ b/internal/cgroup/cgroup_linux.go
@@ -0,0 +1,14 @@
+//go:build linux
+
+package cgroup
+
+import "syscall"
+
+// intoCgroupSupported gates the clone3 CLONE_INTO_CGROUP attempt to
+// platforms whose SysProcAttr has the fields — everywhere else Start goes
+// straight to the PID-move fallback.
+const intoCgroupSupported = true
+
+func sysProcAttrInto(fd int) *syscall.SysProcAttr {
+ return &syscall.SysProcAttr{UseCgroupFD: true, CgroupFD: fd}
+}
diff --git a/internal/cgroup/cgroup_linux_test.go b/internal/cgroup/cgroup_linux_test.go
new file mode 100644
index 000000000..69dd6091d
--- /dev/null
+++ b/internal/cgroup/cgroup_linux_test.go
@@ -0,0 +1,81 @@
+//go:build linux
+
+package cgroup
+
+import (
+ "os"
+ "os/exec"
+ "path/filepath"
+ "syscall"
+ "testing"
+)
+
+// TestStartUsesCgroupFDFirst: with delegation live and no latch, the child
+// starts via clone3-into-cgroup — SysProcAttr set, host dir created, and NO
+// cgroup.procs write (the kernel placed it, no PID move needed).
+func TestStartUsesCgroupFDFirst(t *testing.T) {
+ root := newDelegatedRoot(t)
+ m, err := SetupAt(root, t.Logf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ calls := fakeStart(t, func(*exec.Cmd) error { return nil })
+ if _, err := m.Start("h.example.com", buildTrue); err != nil {
+ t.Fatal(err)
+ }
+ if len(*calls) != 1 {
+ t.Fatalf("started %d times, want 1", len(*calls))
+ }
+ attr := (*calls)[0].SysProcAttr
+ if attr == nil || !attr.UseCgroupFD {
+ t.Fatalf("SysProcAttr = %+v, want UseCgroupFD", attr)
+ }
+ dir := filepath.Join(root, ExecName("h.example.com"))
+ if _, err := os.Stat(dir); err != nil {
+ t.Fatalf("host cgroup dir not created: %v", err)
+ }
+ if _, err := os.Stat(filepath.Join(dir, "cgroup.procs")); !os.IsNotExist(err) {
+ t.Errorf("clone3 path must not write cgroup.procs (the kernel places the child): %v", err)
+ }
+}
+
+// TestStartFallsBackAndLatches pins the refusal path: clone3 refused
+// (seccomp EPERM), the retry WITHOUT the cgroup fd succeeds — proving the
+// cgroup was the problem — the child's PID is moved by file write, and the
+// latch makes every later spawn skip clone3 entirely.
+func TestStartFallsBackAndLatches(t *testing.T) {
+ root := newDelegatedRoot(t)
+ m, err := SetupAt(root, t.Logf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ calls := fakeStart(t, func(cmd *exec.Cmd) error {
+ if cmd.SysProcAttr != nil {
+ return syscall.EPERM // the filter refusing clone3
+ }
+ return nil
+ })
+ if _, err := m.Start("h.example.com", buildTrue); err != nil {
+ t.Fatal(err)
+ }
+ if len(*calls) != 2 {
+ t.Fatalf("first spawn: %d start calls, want 2 (clone3 attempt + plain retry)", len(*calls))
+ }
+ procs := filepath.Join(root, ExecName("h.example.com"), "cgroup.procs")
+ if _, err := os.Stat(procs); err != nil {
+ t.Fatalf("fallback did not move the PID: %v", err)
+ }
+ if !m.fallback.Load() {
+ t.Fatal("refusal did not latch the fallback")
+ }
+
+ if _, err := m.Start("h2.example.com", buildTrue); err != nil {
+ t.Fatal(err)
+ }
+ if len(*calls) != 3 {
+ t.Fatalf("latched spawn: %d total start calls, want 3 (no clone3 re-attempt)", len(*calls))
+ }
+ if (*calls)[2].SysProcAttr != nil {
+ t.Fatal("latched spawn still set SysProcAttr")
+ }
+}
diff --git a/internal/cgroup/cgroup_other.go b/internal/cgroup/cgroup_other.go
new file mode 100644
index 000000000..c865ee827
--- /dev/null
+++ b/internal/cgroup/cgroup_other.go
@@ -0,0 +1,9 @@
+//go:build !linux
+
+package cgroup
+
+import "syscall"
+
+const intoCgroupSupported = false
+
+func sysProcAttrInto(fd int) *syscall.SysProcAttr { return nil }
diff --git a/internal/cgroup/cgroup_test.go b/internal/cgroup/cgroup_test.go
index 3b4736935..10f447212 100644
--- a/internal/cgroup/cgroup_test.go
+++ b/internal/cgroup/cgroup_test.go
@@ -1,7 +1,9 @@
package cgroup
import (
+ "errors"
"os"
+ "os/exec"
"path/filepath"
"strconv"
"testing"
@@ -139,3 +141,86 @@ func TestNilManagerRoot(t *testing.T) {
t.Fatalf("nil Root = %q, want \"\"", got)
}
}
+
+// fakeStart swaps the package's spawn seam for a recorder, restoring it on
+// cleanup. The fake never spawns anything: "success" fabricates cmd.Process
+// from our own PID so the PID-move path has a real number to write.
+func fakeStart(t *testing.T, fail func(cmd *exec.Cmd) error) *[]*exec.Cmd {
+ t.Helper()
+ var calls []*exec.Cmd
+ orig := startCmd
+ startCmd = func(cmd *exec.Cmd) error {
+ calls = append(calls, cmd)
+ if err := fail(cmd); err != nil {
+ return err
+ }
+ cmd.Process, _ = os.FindProcess(os.Getpid())
+ return nil
+ }
+ t.Cleanup(func() { startCmd = orig })
+ return &calls
+}
+
+func buildTrue() *exec.Cmd { return exec.Command("/bin/true") }
+
+// TestStartNilManagerSpawnsPlainly: the nil manager is the no-delegation
+// state — one plain start, no SysProcAttr, no filesystem writes.
+func TestStartNilManagerSpawnsPlainly(t *testing.T) {
+ calls := fakeStart(t, func(*exec.Cmd) error { return nil })
+ var m *Manager
+ cmd, err := m.Start("h.example.com", buildTrue)
+ if err != nil || cmd == nil {
+ t.Fatalf("Start = (%v, %v), want started cmd", cmd, err)
+ }
+ if len(*calls) != 1 || (*calls)[0].SysProcAttr != nil {
+ t.Fatalf("nil manager must start exactly once with no SysProcAttr: %d calls", len(*calls))
+ }
+}
+
+// TestStartGenuineSpawnFailureDoesNotLatch: when the retry-without-cgroup
+// ALSO fails, the cgroup was not the problem — the error propagates and the
+// fallback must not latch (the next spawn should still try clone3).
+func TestStartGenuineSpawnFailureDoesNotLatch(t *testing.T) {
+ root := newDelegatedRoot(t)
+ m, err := SetupAt(root, t.Logf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ fakeStart(t, func(*exec.Cmd) error { return errors.New("no such binary") })
+ if _, err := m.Start("h.example.com", buildTrue); err == nil {
+ t.Fatal("Start with a failing spawn succeeded, want error")
+ }
+ if m.fallback.Load() {
+ t.Fatal("a genuine spawn failure latched the clone3 fallback")
+ }
+}
+
+// TestCleanupHost: removes an empty per-host dir, tolerates a non-empty
+// one, and is nil-safe.
+func TestCleanupHost(t *testing.T) {
+ root := newDelegatedRoot(t)
+ m, err := SetupAt(root, t.Logf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ empty := filepath.Join(root, ExecName("gone.example.com"))
+ if err := os.Mkdir(empty, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ m.CleanupHost("gone.example.com")
+ if _, err := os.Stat(empty); !os.IsNotExist(err) {
+ t.Errorf("empty host dir not removed: %v", err)
+ }
+
+ busy := filepath.Join(root, ExecName("busy.example.com"))
+ if err := os.MkdirAll(filepath.Join(busy, "child"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ m.CleanupHost("busy.example.com") // must not panic or force-remove
+ if _, err := os.Stat(busy); err != nil {
+ t.Errorf("non-empty host dir must survive: %v", err)
+ }
+
+ var nilM *Manager
+ nilM.CleanupHost("h.example.com") // must not panic
+}