amadan

carlos / platform Public

cgroup: Start takes a Limit and writes memory.max/high into the host cgroup (memory-pressure spec §2.3)

Paul Campbell pushed by paul@keymail.dev 36fc91cd773c883ac37f28e3a997530ba1eb63cc parent dd66076
4 files changed, +142 −15
  • internal/activator/backend_exec.go +3 −3
  • internal/cgroup/cgroup.go +48 −3
  • internal/cgroup/cgroup_linux_test.go +7 −7
  • internal/cgroup/cgroup_test.go +84 −2
diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go
index ab062ede4..5bc06d2bb 100644
--- a/internal/activator/backend_exec.go
+++ b/internal/activator/backend_exec.go
@@ -311,7 +311,7 @@ func (b *ExecBackend) Seed(ctx context.Context, rt registry.Route) error {
// cost is that instance's work. No CleanupHost here — the instance spawn
// reuses the dir moments later, and its exit path (or the next agent
// start's sweep) removes it.
- cmd, err := b.Cgroups.Start(host, func() *exec.Cmd {
+ cmd, err := b.Cgroups.Start(host, cgroup.Limit{}, func() *exec.Cmd {
cmd := exec.CommandContext(ctx, b.liveLink(host), argv...)
// The control-channel vars are agent-owned in BOTH directions here too
// (Start's discipline, see withoutVars), and here they are always
@@ -619,7 +619,7 @@ func (b *ExecBackend) Start(ctx context.Context, rt registry.Route) (Proc, strin
// carlos-exec@<host> and the sampler can account it. build may run
// twice — the manager's clone3-refusal fallback — so everything about
// the Cmd is configured inside it.
- cmd, err := b.Cgroups.Start(host, func() *exec.Cmd {
+ cmd, err := b.Cgroups.Start(host, cgroup.Limit{}, func() *exec.Cmd {
cmd := exec.Command(bin, "--socket", rt.Socket, "--db", dbPath)
cmd.Env = childEnv(childVars)
// The child logs through the agent's own stderr/stdout (journald
@@ -822,7 +822,7 @@ func (b *ExecBackend) StartSidecar(_ context.Context, rt registry.Route) (Proc,
// Same host cgroup as the instance: a sidecar's load is that instance's
// work, and attributing it elsewhere would be a lie in the other
// direction.
- cmd, err := b.Cgroups.Start(rt.Host, func() *exec.Cmd {
+ cmd, err := b.Cgroups.Start(rt.Host, cgroup.Limit{}, func() *exec.Cmd {
cmd := exec.Command(b.liveLink(rt.Host), "sidecar", "run")
cmd.Env = childEnv(fileEnv)
// Like the instance child: logs through the agent's own streams
diff --git a/internal/cgroup/cgroup.go b/internal/cgroup/cgroup.go
index 9752a0cf0..64bd89d5c 100644
--- a/internal/cgroup/cgroup.go
+++ b/internal/cgroup/cgroup.go
@@ -3,7 +3,9 @@
// file), so the metrics sampler can report per-instance figures for routes
// that run as exec children of the agent — today those live inside the
// agent's own cgroup and can never show readings. Design:
-// docs/superpowers/specs/2026-08-07-exec-child-accounting-design.md.
+// docs/superpowers/specs/2026-08-07-exec-child-accounting-design.md. Since
+// the memory-pressure spec (2026-08-28) the same per-host cgroup also
+// carries the instance's memory cap (Limit, the tiers).
//
// The agent is NOT root: everything here works through ownership of the
// delegated subtree, never privilege. When delegation is absent (old unit
@@ -40,6 +42,29 @@ func IsExecName(name string) bool {
return strings.HasPrefix(name, execPrefix) && len(name) > len(execPrefix)
}
+// Limit is what Start writes into a host's cgroup before the child lands
+// in it. MemoryMax is bytes; zero means "write nothing" — the pre-spec
+// behaviour for a caller with no tier. memory.high is derived (90% of
+// max) rather than a second field: a tenant that is throttled before it is
+// killed is the whole point, and there is no honest reason for a caller to
+// pick a different band.
+type Limit struct {
+ MemoryMax int64
+}
+
+// The tiers are PLATFORM-chosen (memory-pressure spec §2.3), mirroring
+// what caps/render.go already writes for unit-backed apps: the carlos-app@
+// template's 512M, and the 1G a forking family gets when its manifest
+// declares `children`. No number comes from a manifest.
+var (
+ TierDefault = Limit{MemoryMax: 512 << 20}
+ TierChildren = Limit{MemoryMax: 1 << 30}
+)
+
+// highOf is memory.high for a given memory.max: 90%, so reclaim and
+// throttling start well before the kill.
+func highOf(max int64) int64 { return max / 10 * 9 }
+
// ParseSelfCgroup extracts the cgroup v2 path from /proc/self/cgroup
// content — the "0::" unified-hierarchy line — relative (no leading slash),
// ready to join under /sys/fs/cgroup. Not-ok when there is no v2 line (a
@@ -172,7 +197,8 @@ var startCmd = func(cmd *exec.Cmd) error { return cmd.Start() }
// a seed's first-boot cost and a sidecar's load ARE that instance's work.
func (m *Manager) hostDir(host string) string { return filepath.Join(m.root, ExecName(host)) }
-// Start runs build() and starts the result inside host's cgroup. Nil-safe:
+// Start runs build() and starts the result inside host's cgroup, with
+// limit applied to that cgroup first (memory-pressure spec §2.3). Nil-safe:
// with no manager (or any accounting failure) the child starts exactly as
// today — a spawn must never fail because accounting failed. build must
// return a fully configured, unstarted Cmd; it may be called twice, because
@@ -192,7 +218,7 @@ func (m *Manager) hostDir(host string) string { return filepath.Join(m.root, Exe
// mis-accounting in the safe direction. Note the PID write migrates only
// the child process itself — any grandchildren it has already forked by
// then stay charged to the agent's cgroup, not the child's.
-func (m *Manager) Start(host string, build func() *exec.Cmd) (*exec.Cmd, error) {
+func (m *Manager) Start(host string, limit Limit, build func() *exec.Cmd) (*exec.Cmd, error) {
if m == nil {
cmd := build()
return cmd, startCmd(cmd)
@@ -203,6 +229,7 @@ func (m *Manager) Start(host string, build func() *exec.Cmd) (*exec.Cmd, error)
cmd := build()
return cmd, startCmd(cmd)
}
+ m.applyLimit(host, dir, limit)
if intoCgroupSupported && !m.fallback.Load() {
f, err := os.Open(dir)
if err != nil {
@@ -235,6 +262,24 @@ func (m *Manager) Start(host string, build func() *exec.Cmd) (*exec.Cmd, error)
return m.startAndMove(host, build)
}
+// applyLimit writes limit into dir. Best-effort in the package's usual
+// sense: a refused write (a kernel without the memory controller here, a
+// control file that isn't ours) is one logged line and an uncapped child.
+// Written BEFORE the child is in the cgroup, so the cap holds from its
+// first page. memory.max first: if that one fails, high is pointless.
+func (m *Manager) applyLimit(host, dir string, limit Limit) {
+ if limit.MemoryMax <= 0 {
+ return
+ }
+ if err := os.WriteFile(filepath.Join(dir, "memory.max"), []byte(strconv.FormatInt(limit.MemoryMax, 10)), 0o644); err != nil {
+ m.logf("cgroup: %s: memory.max: %v — child runs uncapped", host, err)
+ return
+ }
+ if err := os.WriteFile(filepath.Join(dir, "memory.high"), []byte(strconv.FormatInt(highOf(limit.MemoryMax), 10)), 0o644); err != nil {
+ m.logf("cgroup: %s: memory.high: %v — child is capped but not throttled first", host, err)
+ }
+}
+
// startAndMove is the fallback spawn: plain start, then write the child's
// PID into the host cgroup. A failed move is logged and tolerated — the
// child runs (unaccounted), which beats not running.
diff --git a/internal/cgroup/cgroup_linux_test.go b/internal/cgroup/cgroup_linux_test.go
index 05d902b58..5abaa2665 100644
--- a/internal/cgroup/cgroup_linux_test.go
+++ b/internal/cgroup/cgroup_linux_test.go
@@ -21,7 +21,7 @@ func TestStartUsesCgroupFDFirst(t *testing.T) {
t.Fatal(err)
}
calls := fakeStart(t, func(*exec.Cmd) error { return nil })
- if _, err := m.Start("h.example.com", buildTrue); err != nil {
+ if _, err := m.Start("h.example.com", Limit{}, buildTrue); err != nil {
t.Fatal(err)
}
if len(*calls) != 1 {
@@ -56,7 +56,7 @@ func TestStartFallsBackAndLatches(t *testing.T) {
}
return nil
})
- if _, err := m.Start("h.example.com", buildTrue); err != nil {
+ if _, err := m.Start("h.example.com", Limit{}, buildTrue); err != nil {
t.Fatal(err)
}
if len(*calls) != 2 {
@@ -70,7 +70,7 @@ func TestStartFallsBackAndLatches(t *testing.T) {
t.Fatal("refusal did not latch the fallback")
}
- if _, err := m.Start("h2.example.com", buildTrue); err != nil {
+ if _, err := m.Start("h2.example.com", Limit{}, buildTrue); err != nil {
t.Fatal(err)
}
if len(*calls) != 3 {
@@ -97,7 +97,7 @@ func TestStartMergesIntoBuilderSysProcAttr(t *testing.T) {
cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true}
return cmd
}
- if _, err := m.Start("h.example.com", build); err != nil {
+ if _, err := m.Start("h.example.com", Limit{}, build); err != nil {
t.Fatal(err)
}
attr := (*calls)[0].SysProcAttr
@@ -126,7 +126,7 @@ func TestStartTransientClone3FailureDoesNotLatch(t *testing.T) {
}
return nil
})
- if _, err := m.Start("h.example.com", buildTrue); err != nil {
+ if _, err := m.Start("h.example.com", Limit{}, buildTrue); err != nil {
t.Fatal(err)
}
if len(*calls) != 2 {
@@ -136,7 +136,7 @@ func TestStartTransientClone3FailureDoesNotLatch(t *testing.T) {
t.Fatal("a transient clone3 failure latched the fallback")
}
- if _, err := m.Start("h2.example.com", buildTrue); err != nil {
+ if _, err := m.Start("h2.example.com", Limit{}, buildTrue); err != nil {
t.Fatal(err)
}
// The fake keeps refusing clone3 with ENODEV for h2 too, so the second
@@ -172,7 +172,7 @@ func TestStartJoinsBothErrorsOnTotalFailure(t *testing.T) {
}
return errRetry
})
- _, err = m.Start("h.example.com", buildTrue)
+ _, err = m.Start("h.example.com", Limit{}, buildTrue)
if err == nil {
t.Fatal("Start with two failing spawns succeeded, want error")
}
diff --git a/internal/cgroup/cgroup_test.go b/internal/cgroup/cgroup_test.go
index ee4ffba26..25debd0d6 100644
--- a/internal/cgroup/cgroup_test.go
+++ b/internal/cgroup/cgroup_test.go
@@ -168,7 +168,7 @@ func buildTrue() *exec.Cmd { return exec.Command("/bin/true") }
func TestStartNilManagerSpawnsPlainly(t *testing.T) {
calls := fakeStart(t, func(*exec.Cmd) error { return nil })
var m *Manager
- cmd, err := m.Start("h.example.com", buildTrue)
+ cmd, err := m.Start("h.example.com", Limit{}, buildTrue)
if err != nil || cmd == nil {
t.Fatalf("Start = (%v, %v), want started cmd", cmd, err)
}
@@ -187,7 +187,7 @@ func TestStartGenuineSpawnFailureDoesNotLatch(t *testing.T) {
t.Fatal(err)
}
fakeStart(t, func(*exec.Cmd) error { return errors.New("no such binary") })
- if _, err := m.Start("h.example.com", buildTrue); err == nil {
+ if _, err := m.Start("h.example.com", Limit{}, buildTrue); err == nil {
t.Fatal("Start with a failing spawn succeeded, want error")
}
if m.fallback.Load() {
@@ -267,3 +267,85 @@ func TestStartAndMoveToleratesBrokenSeam(t *testing.T) {
t.Fatal(err)
}
}
+
+// TestStartWritesMemoryLimit: a non-zero Limit lands in the host cgroup as
+// memory.max and memory.high (high = 90% of max, so a tenant is throttled
+// and reclaimed before it is killed). The temp-dir fixture has no kernel
+// behind it, so the assertion is on the bytes written.
+func TestStartWritesMemoryLimit(t *testing.T) {
+ root := newDelegatedRoot(t)
+ m, err := SetupAt(root, t.Logf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ fakeStart(t, func(*exec.Cmd) error { return nil })
+ if _, err := m.Start("h.example.com", Limit{MemoryMax: 1 << 30}, buildTrue); err != nil {
+ t.Fatal(err)
+ }
+ dir := m.hostDir("h.example.com")
+ for file, want := range map[string]string{
+ "memory.max": "1073741824",
+ "memory.high": "966367638", // max/10*9, integer arithmetic
+ } {
+ got, err := os.ReadFile(filepath.Join(dir, file))
+ if err != nil {
+ t.Fatalf("%s not written: %v", file, err)
+ }
+ if string(got) != want {
+ t.Errorf("%s = %q, want %q", file, got, want)
+ }
+ }
+}
+
+// TestStartZeroLimitWritesNothing: the zero Limit is "no cap" — the
+// pre-spec behaviour, byte for byte, for any caller that has no tier.
+func TestStartZeroLimitWritesNothing(t *testing.T) {
+ root := newDelegatedRoot(t)
+ m, err := SetupAt(root, t.Logf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ fakeStart(t, func(*exec.Cmd) error { return nil })
+ if _, err := m.Start("h.example.com", Limit{}, buildTrue); err != nil {
+ t.Fatal(err)
+ }
+ for _, file := range []string{"memory.max", "memory.high"} {
+ if _, err := os.Stat(filepath.Join(m.hostDir("h.example.com"), file)); !os.IsNotExist(err) {
+ t.Errorf("%s exists for a zero limit", file)
+ }
+ }
+}
+
+// TestStartLimitWriteFailureStillSpawns: a cgroup that refuses the limit
+// (read-only control file, an old kernel) costs one log line and an
+// uncapped child — never a failed spawn. Same discipline as every other
+// accounting failure in this package.
+func TestStartLimitWriteFailureStillSpawns(t *testing.T) {
+ root := newDelegatedRoot(t)
+ m, err := SetupAt(root, t.Logf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ dir := m.hostDir("h.example.com")
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ // A directory where the file should go: WriteFile fails with EISDIR.
+ if err := os.Mkdir(filepath.Join(dir, "memory.max"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ fakeStart(t, func(*exec.Cmd) error { return nil })
+ cmd, err := m.Start("h.example.com", TierDefault, buildTrue)
+ if err != nil || cmd == nil {
+ t.Fatalf("Start = (%v, %v), want a started cmd despite the limit failure", cmd, err)
+ }
+}
+
+func TestTiers(t *testing.T) {
+ if TierDefault.MemoryMax != 512<<20 {
+ t.Errorf("TierDefault = %d, want 512M — the carlos-app@ template's number", TierDefault.MemoryMax)
+ }
+ if TierChildren.MemoryMax != 1<<30 {
+ t.Errorf("TierChildren = %d, want 1G — caps/render.go's children tier", TierChildren.MemoryMax)
+ }
+}