carlos / platform Public

cgroup: the delegation dance — probe, supervise leaf, controllers, sweep

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Paul Campbell 7c59aacf3dc95d9c7c976cf260e10659b12120d3 parent ec7d53d
2 files changed, +212 −2
  • internal/cgroup/cgroup.go +113 −1
  • internal/cgroup/cgroup_test.go +99 −1
diff --git a/internal/cgroup/cgroup.go b/internal/cgroup/cgroup.go
index 7562775a8..b7c27b529 100644
--- a/internal/cgroup/cgroup.go
+++ b/internal/cgroup/cgroup.go
@@ -12,7 +12,14 @@
// a broken spawn path.
package cgroup
-import "strings"
+import (
+ "fmt"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "sync/atomic"
+)
// execPrefix is the ONE definition of the per-host exec cgroup naming —
// activator.ExecName and the sampler's directory match both come through
@@ -50,3 +57,108 @@ func ParseSelfCgroup(data string) (string, bool) {
}
return "", false
}
+
+// Manager owns the delegated subtree. A nil *Manager is the documented
+// "no delegation" state: every method is nil-safe and degrades to today's
+// behavior.
+type Manager struct {
+ root string
+ logf func(string, ...any)
+ // fallback, once latched, means clone3-into-cgroup is refused here (a
+ // seccomp filter older than the syscall, an old kernel) and children are
+ // started normally then moved by PID. A filter is static for the life of
+ // the process, so retrying the syscall on every spawn buys nothing.
+ fallback atomic.Bool
+}
+
+// Root is the delegated subtree's path — the sampler's third scan root.
+// "" for a nil manager (no delegation): the sampler skips an empty root.
+func (m *Manager) Root() string {
+ if m == nil {
+ return ""
+ }
+ return m.root
+}
+
+// Setup arms exec-child accounting for the running agent: resolve our own
+// cgroup from /proc/self/cgroup and run the delegation dance there. Any
+// failure — v1-only box, old unit file without Delegate=, a test, darwin —
+// returns nil with one logged line, and every spawn behaves exactly as
+// today. Fail toward absent readings, never toward a broken spawn path.
+func Setup(logf func(string, ...any)) *Manager {
+ b, err := os.ReadFile("/proc/self/cgroup")
+ if err != nil {
+ logf("agent: exec-child accounting disabled: %v", err)
+ return nil
+ }
+ rel, ok := ParseSelfCgroup(string(b))
+ if !ok {
+ logf("agent: exec-child accounting disabled: no cgroup v2 path in /proc/self/cgroup")
+ return nil
+ }
+ m, err := SetupAt(filepath.Join("/sys/fs/cgroup", rel), logf)
+ if err != nil {
+ logf("agent: exec-child accounting disabled (no delegated subtree — unit file without Delegate=?): %v", err)
+ return nil
+ }
+ logf("agent: exec-child accounting armed (delegated subtree %s)", m.Root())
+ return m
+}
+
+// SetupAt is the delegation dance against an explicit subtree root — the
+// testable core Setup wraps, and the constructor activator tests use over a
+// temp-dir fixture. Order matters: probe, move self to a leaf, THEN enable
+// controllers — cgroup v2 refuses subtree_control writes while the cgroup
+// itself still has member processes ("no internal processes").
+func SetupAt(root string, logf func(string, ...any)) (*Manager, error) {
+ // 1. Probe live delegation: both control files must be writable by us.
+ // systemd chowns exactly these to User= when Delegate= is set; without
+ // it this open is the cheap, side-effect-free "no" that keeps an
+ // undelegated box on today's path.
+ for _, name := range []string{"cgroup.procs", "cgroup.subtree_control"} {
+ f, err := os.OpenFile(filepath.Join(root, name), os.O_WRONLY, 0)
+ if err != nil {
+ return nil, fmt.Errorf("probe delegation: %w", err)
+ }
+ f.Close()
+ }
+ // 2. Move ourselves into a supervise/ leaf, making step 3 legal.
+ sup := filepath.Join(root, "supervise")
+ if err := os.MkdirAll(sup, 0o755); err != nil {
+ return nil, fmt.Errorf("create supervise leaf: %w", err)
+ }
+ if err := os.WriteFile(filepath.Join(sup, "cgroup.procs"), []byte(strconv.Itoa(os.Getpid())), 0o644); err != nil {
+ return nil, fmt.Errorf("move self into supervise leaf: %w", err)
+ }
+ // 3. Enable the controllers per-child cgroups need for memory.current,
+ // io.stat and pids accounting (cpu.stat exists regardless, but +cpu
+ // keeps utilization attributable). One write, all four: the kernel
+ // applies subtree_control writes atomically.
+ if err := os.WriteFile(filepath.Join(root, "cgroup.subtree_control"), []byte("+cpu +memory +io +pids"), 0o644); err != nil {
+ return nil, fmt.Errorf("enable controllers: %w", err)
+ }
+ m := &Manager{root: root, logf: logf}
+ // 4. Sweep per-host dirs a previous agent left behind — empty after a
+ // restart killed their members. rmdir refuses a non-empty dir, which is
+ // exactly the tolerance we want.
+ m.sweepStale()
+ return m, nil
+}
+
+// sweepStale removes leftover carlos-exec@* dirs under the root. Failures
+// are logged and tolerated: a dir that won't rmdir has something alive in
+// it, and killing is the service manager's job, not ours.
+func (m *Manager) sweepStale() {
+ entries, err := os.ReadDir(m.root)
+ if err != nil {
+ return
+ }
+ for _, e := range entries {
+ if !e.IsDir() || !IsExecName(e.Name()) {
+ continue
+ }
+ if err := os.Remove(filepath.Join(m.root, e.Name())); err != nil {
+ m.logf("cgroup: stale %s not removed: %v", e.Name(), err)
+ }
+ }
+}
diff --git a/internal/cgroup/cgroup_test.go b/internal/cgroup/cgroup_test.go
index cba5f40da..3b4736935 100644
--- a/internal/cgroup/cgroup_test.go
+++ b/internal/cgroup/cgroup_test.go
@@ -1,6 +1,11 @@
package cgroup
-import "testing"
+import (
+ "os"
+ "path/filepath"
+ "strconv"
+ "testing"
+)
func TestExecNameAndIsExecName(t *testing.T) {
if got := ExecName("keymail.dev"); got != "carlos-exec@keymail.dev" {
@@ -41,3 +46,96 @@ func TestParseSelfCgroup(t *testing.T) {
}
}
}
+
+// newDelegatedRoot builds the temp-dir shape of a delegated cgroup: the two
+// control files systemd chowns to the unit's user when Delegate= is set.
+// Everything the dance does is plain file I/O, so a temp dir is a faithful
+// fixture — the same seam listCarlosUnits uses in internal/host.
+func newDelegatedRoot(t *testing.T) string {
+ t.Helper()
+ root := t.TempDir()
+ for _, f := range []string{"cgroup.procs", "cgroup.subtree_control"} {
+ if err := os.WriteFile(filepath.Join(root, f), nil, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return root
+}
+
+// TestSetupAtRefusesUndelegatedRoot: without Delegate= the control files are
+// root-owned (here: absent) — SetupAt must error so Setup degrades to the
+// nil manager instead of half-arming.
+func TestSetupAtRefusesUndelegatedRoot(t *testing.T) {
+ if _, err := SetupAt(t.TempDir(), t.Logf); err == nil {
+ t.Fatal("SetupAt on an undelegated root succeeded, want error")
+ }
+}
+
+// TestSetupAtDance pins the dance's observable results: the agent's own PID
+// moved into the supervise/ leaf (cgroup v2 forbids enabling controllers on
+// a cgroup with member processes), then the four controllers enabled in
+// subtree_control, and Root reporting the subtree.
+func TestSetupAtDance(t *testing.T) {
+ root := newDelegatedRoot(t)
+ m, err := SetupAt(root, t.Logf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if m.Root() != root {
+ t.Errorf("Root = %q, want %q", m.Root(), root)
+ }
+ pid, err := os.ReadFile(filepath.Join(root, "supervise", "cgroup.procs"))
+ if err != nil {
+ t.Fatalf("supervise leaf not populated: %v", err)
+ }
+ if string(pid) != strconv.Itoa(os.Getpid()) {
+ t.Errorf("supervise/cgroup.procs = %q, want our own pid %d", pid, os.Getpid())
+ }
+ ctrl, err := os.ReadFile(filepath.Join(root, "cgroup.subtree_control"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if string(ctrl) != "+cpu +memory +io +pids" {
+ t.Errorf("subtree_control = %q, want %q", ctrl, "+cpu +memory +io +pids")
+ }
+}
+
+// TestSetupAtSweepsStaleExecDirs: a previous agent's per-host dirs are empty
+// after its children died with the service; the dance removes them. A
+// non-empty one (something still alive in it — shouldn't happen, but rmdir
+// would fail anyway) and a foreign name are left alone.
+func TestSetupAtSweepsStaleExecDirs(t *testing.T) {
+ root := newDelegatedRoot(t)
+ stale := filepath.Join(root, "carlos-exec@old.example.com")
+ if err := os.Mkdir(stale, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ occupied := filepath.Join(root, "carlos-exec@busy.example.com")
+ if err := os.MkdirAll(filepath.Join(occupied, "child"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ foreign := filepath.Join(root, "not-ours")
+ if err := os.Mkdir(foreign, 0o755); err != nil {
+ t.Fatal(err)
+ }
+
+ if _, err := SetupAt(root, t.Logf); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(stale); !os.IsNotExist(err) {
+ t.Errorf("stale exec dir survived the sweep: %v", err)
+ }
+ if _, err := os.Stat(occupied); err != nil {
+ t.Errorf("non-empty exec dir must be tolerated, not force-removed: %v", err)
+ }
+ if _, err := os.Stat(foreign); err != nil {
+ t.Errorf("foreign dir must be untouched: %v", err)
+ }
+}
+
+func TestNilManagerRoot(t *testing.T) {
+ var m *Manager
+ if got := m.Root(); got != "" {
+ t.Fatalf("nil Root = %q, want \"\"", got)
+ }
+}