amadan
cgroup: a generation whose limits cannot be applied does not start
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
21 files changed,
+776
−48
internal/activator/claimspawn_kernel_linux_test.go+31 −2internal/activator/containment_test.go+1 −0internal/activator/exec_gate_test.go+16 −14internal/activator/genbackend_test.go+1 −0internal/activator/genparent_kernel_linux_test.go+140 −0internal/activator/identity_test.go+3 −0internal/activator/logsink.go+14 −4internal/activator/logsink_test.go+18 −0internal/activator/opset_test.go+1 −0internal/activator/swap.go+8 −2internal/activator/swap_executed_test.go+9 −0internal/activator/swap_overlap_test.go+8 −0internal/activator/swap_sidecar_test.go+8 −2internal/cgroup/cgroup.go+80 −7internal/cgroup/cgroup_linux.go+21 −0internal/cgroup/cgroup_linux_test.go+79 −0internal/cgroup/cgroup_other.go+4 −0internal/cgroup/gen.go+65 −14internal/cgroup/gen_kernel_linux_test.go+4 −3internal/cgroup/gen_test.go+264 −0internal/cgroup/recreate_linux_test.go+1 −0
diff --git a/internal/activator/claimspawn_kernel_linux_test.go b/internal/activator/claimspawn_kernel_linux_test.go| index 0041f9029..849b5aa8a 100644 |
| --- a/internal/activator/claimspawn_kernel_linux_test.go |
| +++ b/internal/activator/claimspawn_kernel_linux_test.go |
| @@ -9,6 +9,7 @@ import ( |
| "os" |
| "os/exec" |
| "path/filepath" |
| + "slices" |
| "strconv" |
| "strings" |
| "syscall" |
| @@ -179,7 +180,11 @@ func killHostRespawn(t *testing.T, manager *cgroup.Manager, build func() *exec.C |
| // killSidecarRespawn is the legacy sidecar's sequence under containment: a |
| // flat row's sidecar spawned into carlos-exec@<host>/role-sidecar, proven |
| // gone by the real ExecBackend.KillSidecar, then respawned. Each respawn gets |
| -// a fresh role-sidecar under the same parent, and stays up. |
| +// a fresh role-sidecar under the same parent, and stays up. A live g1 runs |
| +// under the parent throughout (zdd readiness round 2, Task 7): a parent |
| +// populated only through its children is no legacy parent, so no respawn |
| +// is refused, the parent never holds a process directly, it keeps memory |
| +// delegated, and g1 is untouched by every kill. |
| func killSidecarRespawn(t *testing.T, manager *cgroup.Manager, build func() *exec.Cmd) { |
| const host = "killsidecar.claimspawn.example" |
| b := &ExecBackend{Cgroups: manager, GenContainment: true, DataDir: t.TempDir(), OptDir: t.TempDir()} |
| @@ -187,11 +192,23 @@ func killSidecarRespawn(t *testing.T, manager *cgroup.Manager, build func() *exe |
| Socket: filepath.Join(provision.SocketDir, host+".sock")} |
| parent := filepath.Join(manager.Root(), cgroup.ExecName(host)) |
| child := cgroup.RoleChild("sidecar") |
| + g1, _, err := b.spawnInCgroupAt(rt, cgroup.GenChild(1), true, build) |
| + if err != nil || g1 == nil { |
| + t.Fatalf("live g1: %v, %v", g1, err) |
| + } |
| + t.Cleanup(func() { _ = g1.Process.Kill() }) |
| + g1Done := spawnStaysUp(t, "live g1", g1) |
| var last, parentIno uint64 |
| for attempt := range 3 { |
| cmd, settle, err := b.spawnInCgroupAt(rt, child, false, build) |
| if err != nil { |
| - t.Fatalf("spawn %d: %v", attempt, err) |
| + t.Fatalf("spawn %d beside a live g1: %v", attempt, err) |
| + } |
| + if got, err := os.ReadFile(filepath.Join(parent, "cgroup.procs")); err != nil || strings.TrimSpace(string(got)) != "" { |
| + t.Fatalf("spawn %d: the parent holds processes directly: %q, %v", attempt, got, err) |
| + } |
| + if got, err := os.ReadFile(filepath.Join(parent, "cgroup.subtree_control")); err != nil || !slices.Contains(strings.Fields(string(got)), "memory") { |
| + t.Fatalf("spawn %d: the parent's subtree_control is %q, %v; want memory delegated", attempt, got, err) |
| } |
| done := spawnStaysUp(t, fmt.Sprintf("sidecar spawn %d", attempt), cmd) |
| ino := cgroupInode(t, filepath.Join(parent, child)) |
| @@ -213,6 +230,18 @@ func killSidecarRespawn(t *testing.T, manager *cgroup.Manager, build func() *exe |
| reaped(t, fmt.Sprintf("sidecar spawn %d", attempt), done) |
| settle() |
| } |
| + select { |
| + case err := <-g1Done: |
| + t.Fatalf("the live g1 died during the sidecar respawns: %v", err) |
| + default: |
| + } |
| + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) |
| + err = b.KillGen(ctx, rt, 1) |
| + cancel() |
| + if err != nil { |
| + t.Fatalf("KillGen 1: %v", err) |
| + } |
| + reaped(t, "live g1", g1Done) |
| manager.RemoveChild(host, child) |
| manager.CleanupHost(host) |
| } |
diff --git a/internal/activator/containment_test.go b/internal/activator/containment_test.go| index 57bb6017b..993471a27 100644 |
| --- a/internal/activator/containment_test.go |
| +++ b/internal/activator/containment_test.go |
| @@ -107,6 +107,7 @@ func delegatedBackend(t *testing.T, host string) (*ExecBackend, string) { |
| func TestSpawnInCgroupAtPutsAGenerationUnderItsParent(t *testing.T) { |
| const host = "h.example.com" |
| b, parent := delegatedBackend(t, host) |
| + cgroup.FakePlacementForTest(t) // a temp dir is no cgroupfs, and a generation is never moved by PID |
| b.GenContainment = true |
| cmd, _, err := b.spawnInCgroupAt(registry.Route{Host: host}, "g7", true, func() *exec.Cmd { return exec.Command("/bin/true") }) |
| if err != nil { |
diff --git a/internal/activator/exec_gate_test.go b/internal/activator/exec_gate_test.go| index 4fb87823f..33da67a70 100644 |
| --- a/internal/activator/exec_gate_test.go |
| +++ b/internal/activator/exec_gate_test.go |
| @@ -566,36 +566,38 @@ func TestTheExecGateIsAskedAtEachCgroupStart(t *testing.T) { |
| } |
| b.prepGone(t, b.gens()[0]) |
| }) |
| - t.Run("overlap successor, superseded between a failed start and its retry", func(t *testing.T) { |
| + t.Run("overlap successor, a failed clone3 start is not retried by PID move", func(t *testing.T) { |
| f := newSwapFixture(t) |
| b := newLaunchBackend(t, "flat", f.b) |
| b.eb.GenContainment = true |
| // The temp-dir subtree is no cgroupfs: the clone3-into-cgroup start |
| - // fails, and the manager retries with the PID move. |
| - gatedCgroups(t, b.eb, b.rt.Host, nil) |
| + // fails (EBADF). A generation is never started and then moved by |
| + // PID (readiness round 2 Task 7), so there is no retry and no second |
| + // gate: the start is refused as the box's condition. |
| + parent := gatedCgroups(t, b.eb, b.rt.Host, nil) |
| var asked atomic.Int32 |
| - b.eb.launchHook = func(string) { |
| - if asked.Add(1) == 2 { |
| - f.rev.Add(1) |
| - } |
| - } |
| + b.eb.launchHook = func(string) { asked.Add(1) } |
| f.a.backend = b |
| mark := filepath.Join(t.TempDir(), "v2.ran") |
| id := markerScript(t, t.TempDir(), "v2", mark) |
| at := f.attempt(ModeOverlap) |
| at.StagedPath, at.Digest = id.StagedPath, id.Digest |
| res := f.a.Swap(context.Background(), at, f.newest) |
| - if n := asked.Load(); n != 2 { |
| - t.Fatalf("the gate was asked %d times, want twice: the clone3 start and its retry", n) |
| + if n := asked.Load(); n != 1 { |
| + t.Fatalf("the gate was asked %d times, want once: the clone3 start, and no retry", n) |
| } |
| - if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, errSwapSuperseded.Error()) || res.Executed { |
| - t.Fatalf("Swap = %+v, want abandoned (superseded), nothing executed", res) |
| + if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, cgroup.ErrLimitNotApplied.Error()) || res.Executed { |
| + t.Fatalf("Swap = %+v, want abandoned (limits not applied), nothing executed", res) |
| } |
| time.Sleep(100 * time.Millisecond) |
| if b.launched(t, id, mark) { |
| - t.Fatal("the successor was launched by the retry after its attempt was superseded") |
| + t.Fatal("the successor was launched after its clone3 placement failed") |
| + } |
| + gens := b.gens() |
| + b.prepGone(t, gens[0]) |
| + if got, err := os.ReadFile(filepath.Join(parent, cgroup.GenChild(gens[0]), "cgroup.procs")); err == nil && len(got) != 0 { |
| + t.Fatalf("generation %d's child cgroup was given pid %s by a PID move", gens[0], got) |
| } |
| - b.prepGone(t, b.gens()[0]) |
| }) |
| } |
diff --git a/internal/activator/genbackend_test.go b/internal/activator/genbackend_test.go| index a25323035..1847e5a97 100644 |
| --- a/internal/activator/genbackend_test.go |
| +++ b/internal/activator/genbackend_test.go |
| @@ -245,6 +245,7 @@ func TestExecBackendStartStagedClearsALeftoverCanonicalLink(t *testing.T) { |
| func TestExecBackendUsesEachArtifactsTier(t *testing.T) { |
| const host = "t.example.com" |
| b, parent := delegatedBackend(t, host) |
| + cgroup.FakePlacementForTest(t) // a temp dir is no cgroupfs, and a generation is never moved by PID |
| b.GenContainment = true |
| b.Tier = func(registry.Route) cgroup.Limit { return cgroup.TierDefault } |
| b.TierOf = func(_ registry.Route, v string) cgroup.Limit { |
diff --git a/internal/activator/genparent_kernel_linux_test.go b/internal/activator/genparent_kernel_linux_test.go| index 19b11a573..df9b0f111 100644 |
| --- a/internal/activator/genparent_kernel_linux_test.go |
| +++ b/internal/activator/genparent_kernel_linux_test.go |
| @@ -3,14 +3,20 @@ |
| package activator |
| import ( |
| + "context" |
| "errors" |
| + "fmt" |
| "os" |
| "os/exec" |
| "path/filepath" |
| + "slices" |
| + "strconv" |
| "strings" |
| "testing" |
| + "time" |
| "github.com/carlosframework/platform/internal/cgroup" |
| + "github.com/carlosframework/platform/internal/provision" |
| "github.com/carlosframework/platform/internal/registry" |
| ) |
| @@ -78,4 +84,138 @@ func TestGenerationStartRefusedBesideADirectParentMember(t *testing.T) { |
| t.Errorf("the refused start left %s under the parent", e.Name()) |
| } |
| } |
| + |
| + // Positive controls (zdd readiness round 2, Task 7): the refusal above |
| + // says nothing unless the same start succeeds, capped, where the parent |
| + // holds no process directly. The sibling control runs on another host; |
| + // the recovery control is this host once its direct member is gone. |
| + t.Run("sibling control", func(t *testing.T) { genParentSiblingControl(t, manager) }) |
| + t.Run("recovery control", func(t *testing.T) { genParentRecoveryControl(t, b, host, parent, legacy) }) |
| +} |
| + |
| +// genParentKernelChild is the sleeper TestClaimSpawnKernelChild runs as. |
| +func genParentKernelChild(t *testing.T) func() *exec.Cmd { |
| + dir := t.TempDir() |
| + return func() *exec.Cmd { |
| + cmd := exec.Command(os.Args[0], "-test.run=^TestClaimSpawnKernelChild$", "-test.count=1") |
| + cmd.Env = []string{"CARLOS_CLAIM_SPAWN_CHILD=1"} |
| + cmd.Dir = dir |
| + return cmd |
| + } |
| +} |
| + |
| +// readCgroupFile reads one cgroupfs file, trimmed. |
| +func readCgroupFile(t *testing.T, path string) string { |
| + t.Helper() |
| + b, err := os.ReadFile(path) |
| + if err != nil { |
| + t.Fatalf("read %s: %v", path, err) |
| + } |
| + return strings.TrimSpace(string(b)) |
| +} |
| + |
| +// expectGenParent checks a generation parent the layout made: no process |
| +// of its own, memory delegated to its children, memory.oom.group=0. |
| +func expectGenParent(t *testing.T, parent string) { |
| + t.Helper() |
| + if got := readCgroupFile(t, filepath.Join(parent, "cgroup.procs")); got != "" { |
| + t.Errorf("the parent holds processes directly: %q", got) |
| + } |
| + if got := readCgroupFile(t, filepath.Join(parent, "cgroup.subtree_control")); !slices.Contains(strings.Fields(got), "memory") { |
| + t.Errorf("the parent's subtree_control is %q, want memory delegated", got) |
| + } |
| + if got := readCgroupFile(t, filepath.Join(parent, "memory.oom.group")); got != "0" { |
| + t.Errorf("the parent's memory.oom.group = %q, want 0", got) |
| + } |
| +} |
| + |
| +// expectCappedGen checks that cmd runs in host's parent/<child> and that the |
| +// child carries the default tier. |
| +func expectCappedGen(t *testing.T, host, parent, child string, cmd *exec.Cmd) { |
| + t.Helper() |
| + tier := strconv.FormatInt(cgroup.TierDefault.MemoryMax, 10) |
| + if got := readCgroupFile(t, filepath.Join(parent, child, "memory.max")); got != tier { |
| + t.Errorf("%s memory.max = %q, want the tier %s", child, got, tier) |
| + } |
| + own, ok := cgroup.ParseSelfCgroup(readCgroupFile(t, filepath.Join("/proc", strconv.Itoa(cmd.Process.Pid), "cgroup"))) |
| + if want := "/" + cgroup.ExecName(host) + "/" + child; !ok || !strings.HasSuffix(own, want) { |
| + t.Errorf("pid %d runs in %q, want .../%s", cmd.Process.Pid, own, want[1:]) |
| + } |
| +} |
| + |
| +// genParentSiblingControl: on a host whose parent holds no process, g1 |
| +// starts capped, and g2 starts capped beside the live g1 once the parent is |
| +// raised, as a swap's overlap does. |
| +func genParentSiblingControl(t *testing.T, manager *cgroup.Manager) { |
| + const host = "genparent-ok.example" |
| + b := &ExecBackend{Cgroups: manager, GenContainment: true, DataDir: t.TempDir(), OptDir: t.TempDir()} |
| + rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.DefaultDBPath(b.DataDir, host), |
| + Socket: filepath.Join(provision.SocketDir, host+".sock")} |
| + parent := filepath.Join(manager.Root(), cgroup.ExecName(host)) |
| + t.Cleanup(func() { manager.CleanupHost(host) }) // registered first, so it runs last |
| + build := genParentKernelChild(t) |
| + |
| + g1, _, err := b.spawnInCgroupAt(rt, cgroup.GenChild(1), true, build) |
| + if err != nil || g1 == nil { |
| + t.Fatalf("g1 under an empty parent: %v, %v", g1, err) |
| + } |
| + t.Cleanup(func() { _ = g1.Process.Kill() }) |
| + g1Done := spawnStaysUp(t, "g1", g1) |
| + expectGenParent(t, parent) |
| + expectCappedGen(t, host, parent, cgroup.GenChild(1), g1) |
| + |
| + if err := manager.SetParentLimit(host, cgroup.OverlapLimit(cgroup.TierDefault, cgroup.TierDefault)); err != nil { |
| + t.Fatal(err) |
| + } |
| + g2, _, err := b.spawnInCgroupAt(rt, cgroup.GenChild(2), true, build) |
| + if err != nil || g2 == nil { |
| + t.Fatalf("g2 beside a live g1: %v, %v", g2, err) |
| + } |
| + t.Cleanup(func() { _ = g2.Process.Kill() }) |
| + g2Done := spawnStaysUp(t, "g2", g2) |
| + expectGenParent(t, parent) |
| + expectCappedGen(t, host, parent, cgroup.GenChild(2), g2) |
| + select { |
| + case err := <-g1Done: |
| + t.Fatalf("g1 died while g2 started beside it: %v", err) |
| + default: |
| + } |
| + |
| + for _, g := range []struct { |
| + n uint64 |
| + done <-chan error |
| + }{{2, g2Done}, {1, g1Done}} { |
| + ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) |
| + err := b.KillGen(ctx, rt, g.n) |
| + cancel() |
| + if err != nil { |
| + t.Fatalf("KillGen %d: %v", g.n, err) |
| + } |
| + reaped(t, fmt.Sprintf("g%d", g.n), g.done) |
| + } |
| +} |
| + |
| +// genParentRecoveryControl: once the direct member the refusal named has |
| +// exited, the same host starts a capped g1 under a parent the layout made. |
| +// This is the manual recovery amadan #272's ruling relies on. |
| +func genParentRecoveryControl(t *testing.T, b *ExecBackend, host, parent string, legacy *exec.Cmd) { |
| + _ = legacy.Process.Kill() |
| + _ = legacy.Wait() |
| + deadline := time.Now().Add(5 * time.Second) |
| + for readCgroupFile(t, filepath.Join(parent, "cgroup.procs")) != "" { |
| + if time.Now().After(deadline) { |
| + t.Fatal("the parent still holds the killed legacy process") |
| + } |
| + time.Sleep(20 * time.Millisecond) |
| + } |
| + g1, _, err := b.spawnInCgroupAt(registry.Route{Host: host}, cgroup.GenChild(1), true, genParentKernelChild(t)) |
| + if err != nil || g1 == nil { |
| + t.Fatalf("g1 after the legacy process exited: %v, %v", g1, err) |
| + } |
| + done := spawnStaysUp(t, "recovered g1", g1) |
| + expectGenParent(t, parent) |
| + expectCappedGen(t, host, parent, cgroup.GenChild(1), g1) |
| + _ = g1.Process.Kill() |
| + reaped(t, "recovered g1", done) |
| + b.Cgroups.RemoveChild(host, cgroup.GenChild(1)) |
| } |
diff --git a/internal/activator/identity_test.go b/internal/activator/identity_test.go| index b3e580090..c564813bb 100644 |
| --- a/internal/activator/identity_test.go |
| +++ b/internal/activator/identity_test.go |
| @@ -764,6 +764,9 @@ func TestStartGenExeAndSeedExeUseTheRecordedBuildsTier(t *testing.T) { |
| b, parent := delegatedBackend(t, host) |
| b.RunDirOverride = shortRunDir(t) |
| b.GenContainment = gens |
| + if gens { |
| + cgroup.FakePlacementForTest(t) // a temp dir is no cgroupfs, and a generation is never moved by PID |
| + } |
| tierFollowsTheLink(b, "vA") |
| marker := filepath.Join(t.TempDir(), "argv0") |
| stageVersion(t, b, host, "vA", servingBody(t, marker)) |
diff --git a/internal/activator/logsink.go b/internal/activator/logsink.go| index ec5622a07..049dd0cd6 100644 |
| --- a/internal/activator/logsink.go |
| +++ b/internal/activator/logsink.go |
| @@ -85,10 +85,20 @@ func memberSafeWakeReason(err error) string { |
| if errors.Is(err, cgroup.ErrLegacyParentPopulated) { |
| // A generation start refused because the host's parent cgroup still |
| // holds a process directly (amadan #272). It is the box's state to |
| - // clear, and the cgroup detail (the host, the count) is the |
| - // operator's: the agent logs it. The member gets the maintenance |
| - // fence's existing text, as for the other holds the platform lifts |
| - // itself. |
| + // clear: the platform does not lift it by itself; the process must |
| + // exit or be stopped (or the flag go off). The cgroup detail (the |
| + // host, the count) is the operator's: the agent logs it. The member |
| + // gets the maintenance fence's existing text, the box's condition |
| + // in the words the platform already uses for it. |
| + err = ErrFenced |
| + } |
| + if errors.Is(err, cgroup.ErrLimitNotApplied) { |
| + // A generation start refused because a limit it would run under |
| + // could not be applied, or it could only be started outside its |
| + // cgroup and moved after (zdd readiness round 2, Task 7). Like the |
| + // populated parent above it is the box's condition, and the cgroup |
| + // detail is the operator's (the manager logs it), so the member |
| + // gets the maintenance fence's existing text. |
| err = ErrFenced |
| } |
| if errors.Is(err, cgroup.ErrKilledCgroupKept) { |
diff --git a/internal/activator/logsink_test.go b/internal/activator/logsink_test.go| index 344a72936..15e0530f6 100644 |
| --- a/internal/activator/logsink_test.go |
| +++ b/internal/activator/logsink_test.go |
| @@ -222,3 +222,21 @@ func TestMemberSafeWakeReasonHidesAKilledCgroupRefusal(t *testing.T) { |
| t.Error("a start refused for a killed cgroup counted as one that may have run") |
| } |
| } |
| + |
| +// Task 7 (readiness round 2): a wake refused because its generation's limits |
| +// could not be applied reads to a member as the maintenance fence's approved |
| +// text, not as cgroup detail. Nothing of the build ran. |
| +func TestMemberSafeWakeReasonHidesALimitNotAppliedRefusal(t *testing.T) { |
| + for _, err := range []error{ |
| + fmt.Errorf("h.example.com: %w: g2: memory.max: open /sys/fs/cgroup/x/carlos-exec@h.example.com/g2/memory.max: permission denied", cgroup.ErrLimitNotApplied), |
| + fmt.Errorf("%w: h.example.com: clone3-into-cgroup unavailable", cgroup.ErrLimitNotApplied), |
| + } { |
| + got := memberSafeWakeReason(err) |
| + if got != memberSafeWakeReason(ErrFenced) || strings.Contains(got, "cgroup") || strings.Contains(got, "memory") { |
| + t.Errorf("member reason = %q, want the fence text alone", got) |
| + } |
| + if startMayHaveRun(nil, err) { |
| + t.Error("a start refused for its limits counted as one that may have run") |
| + } |
| + } |
| +} |
diff --git a/internal/activator/opset_test.go b/internal/activator/opset_test.go| index 8018503f9..59b8b5af9 100644 |
| --- a/internal/activator/opset_test.go |
| +++ b/internal/activator/opset_test.go |
| @@ -682,6 +682,7 @@ func TestAPlacedSpawnWithAFailedHealthWaitProvesItsChild(t *testing.T) { |
| t.Run(row.name, func(t *testing.T) { |
| host := "placed.example.com" |
| x, _ := delegatedBackend(t, host) |
| + cgroup.FakePlacementForTest(t) // a temp dir is no cgroupfs, and a generation is never moved by PID |
| x.GenContainment = row.containment |
| x.RunDirOverride = shortRunDir(t) |
| stageLive(t, x, host, "v1") // exits at once without binding: the health wait fails |
diff --git a/internal/activator/swap.go b/internal/activator/swap.go| index 93079aeb5..8365047c2 100644 |
| --- a/internal/activator/swap.go |
| +++ b/internal/activator/swap.go |
| @@ -182,7 +182,7 @@ const ( |
| // verdictOf types an attempt-gate error. An error that is not the gate's |
| // (a failure of the build) is gateProceed: the gate did not refuse. |
| // |
| -// Besides the gate's own refusals, six kinds of refusal that stop a spawn |
| +// Besides the gate's own refusals, eight kinds of refusal that stop a spawn |
| // before anything of the build runs are NoOwnership, so the one classifier |
| // reads them as abandonment, never as a failed build: |
| // - plan 2's strict-host gate on placed spawns (index P2-8): |
| @@ -200,6 +200,12 @@ const ( |
| // into took cgroup.kill and could not be made fresh |
| // (cgroup.ErrKilledCgroupKept, amadan #272, Task 19): it refuses before |
| // it builds the command, so nothing ran, and the condition is the box's; |
| +// - the cgroup manager refusing a generation start because a limit it |
| +// would run under could not be applied, or it could only be started |
| +// outside its cgroup and moved after (cgroup.ErrLimitNotApplied, zdd |
| +// readiness round 2 Task 7): it refuses before it builds the command |
| +// (or, for a failed clone3 placement, the kernel refused the clone), |
| +// so nothing ran, and the condition is the box's; |
| // - root refusing the nested staged executable (instancectl.ErrStagedExe, |
| // Task 9): it validates before it mounts or starts anything; |
| // - a build state that could not be written before the exec |
| @@ -231,7 +237,7 @@ func verdictOf(err error) gateVerdict { |
| errors.Is(err, hostbox.ErrChanged), errors.Is(err, errClaimsAdmission), |
| errors.Is(err, ErrStrictRequired), errors.Is(err, ErrStrictGroupInUse), errors.Is(err, ErrStrictCleanupPending), |
| errors.Is(err, ErrClaimIsolationUnavailable), errors.Is(err, cgroup.ErrLegacyParentPopulated), |
| - errors.Is(err, cgroup.ErrKilledCgroupKept), |
| + errors.Is(err, cgroup.ErrKilledCgroupKept), errors.Is(err, cgroup.ErrLimitNotApplied), |
| errors.Is(err, errUnitNoGens), errors.Is(err, errSwapNoGens), errors.Is(err, instancectl.ErrStagedExe), |
| errors.Is(err, errBuildStateUnwritten): |
| return gateNoOwnership |
diff --git a/internal/activator/swap_executed_test.go b/internal/activator/swap_executed_test.go| index e77f5794d..4dbd5dc22 100644 |
| --- a/internal/activator/swap_executed_test.go |
| +++ b/internal/activator/swap_executed_test.go |
| @@ -54,6 +54,15 @@ func TestSwapResultReportsWhetherTheAttemptsBuildExecuted(t *testing.T) { |
| {"hold, a killed cgroup would not go", ModeHold, func(f *swapFixture) { |
| f.b.genErr = fmt.Errorf("%w: carlos-exec@h.example.com: device or resource busy", cgroup.ErrKilledCgroupKept) |
| }, SwapAbandoned, false}, |
| + // Task 7 (readiness round 2): the successor's limits could not be |
| + // applied, so the manager refused before it built the command: the |
| + // box's condition, nothing ran, the allowance is not spent. |
| + {"overlap, the generation's limits were not applied", ModeOverlap, func(f *swapFixture) { |
| + f.b.genErr = fmt.Errorf("%w: g2: memory.max: is a directory", cgroup.ErrLimitNotApplied) |
| + }, SwapAbandoned, false}, |
| + {"hold, the generation's limits were not applied", ModeHold, func(f *swapFixture) { |
| + f.b.genErr = fmt.Errorf("%w: h.example.com: clone3-into-cgroup unavailable", cgroup.ErrLimitNotApplied) |
| + }, SwapAbandoned, false}, |
| {"overlap, superseded at the switch", ModeOverlap, func(f *swapFixture) { |
| f.a.genHealth = func(context.Context, Gen) error { f.rev.Add(1); return nil } |
| }, SwapAbandoned, true}, |
diff --git a/internal/activator/swap_overlap_test.go b/internal/activator/swap_overlap_test.go| index c9cdce781..da60172d1 100644 |
| --- a/internal/activator/swap_overlap_test.go |
| +++ b/internal/activator/swap_overlap_test.go |
| @@ -297,6 +297,9 @@ func TestClassifyStep(t *testing.T) { |
| // Task 19: the start's killed cgroup could not be made fresh, so the |
| // manager refused before any exec. |
| {"a killed cgroup that would not go", live, fmt.Errorf("%w: carlos-exec@h/g2: device or resource busy", cgroup.ErrKilledCgroupKept), stepStale}, |
| + // Task 7 (readiness round 2): a generation whose limits could not be |
| + // applied is refused before any exec: the box's condition. |
| + {"a generation's limits not applied", live, fmt.Errorf("h: %w: g2: memory.max: is a directory", cgroup.ErrLimitNotApplied), stepStale}, |
| // Root refused the nested staged executable before it mounted or |
| // started anything (Task 9): an abandonment, not a failed build. |
| {"root refused the staged exe", live, fmt.Errorf("start unit: %w: not blog's staged v2", instancectl.ErrStagedExe), stepStale}, |
| @@ -354,6 +357,11 @@ func TestVerdictOf(t *testing.T) { |
| // refused before the command is built; no build is at fault. |
| {cgroup.ErrKilledCgroupKept, gateNoOwnership}, |
| {fmt.Errorf("%w: carlos-exec@h/role-sidecar: device or resource busy", cgroup.ErrKilledCgroupKept), gateNoOwnership}, |
| + // Task 7: a generation start refused because its limits could not |
| + // all be applied (or it could only be started then moved) is refused |
| + // before the exec; no build is at fault. |
| + {cgroup.ErrLimitNotApplied, gateNoOwnership}, |
| + {fmt.Errorf("h: %w: clone3-into-cgroup unavailable", cgroup.ErrLimitNotApplied), gateNoOwnership}, |
| // A row that cannot run a staged generation: nothing staged spawns. |
| {errUnitNoGens, gateNoOwnership}, |
| {instancectl.ErrStagedExe, gateNoOwnership}, |
diff --git a/internal/activator/swap_sidecar_test.go b/internal/activator/swap_sidecar_test.go| index c27769f9e..1b0832e67 100644 |
| --- a/internal/activator/swap_sidecar_test.go |
| +++ b/internal/activator/swap_sidecar_test.go |
| @@ -11,6 +11,7 @@ import ( |
| "testing" |
| "time" |
| + "github.com/carlosframework/platform/internal/cgroup" |
| "github.com/carlosframework/platform/internal/provision" |
| "github.com/carlosframework/platform/internal/registry" |
| ) |
| @@ -568,12 +569,14 @@ func TestSweepReplacesASidecarLeftOnARetiredGeneration(t *testing.T) { |
| func TestStartSidecarGenKeepsTheRoleSidecarChild(t *testing.T) { |
| const host = "sc.example.com" |
| b, parent := delegatedBackend(t, host) |
| + cgroup.FakePlacementForTest(t) // a temp dir is no cgroupfs, and a generation is never moved by PID |
| b.GenContainment = true |
| v2 := filepath.Join(b.OptDir, "versions", "v2", "app") |
| if err := os.MkdirAll(filepath.Dir(v2), 0o755); err != nil { |
| t.Fatal(err) |
| } |
| - if err := os.WriteFile(v2, []byte("#!/bin/sh\nexit 0\n"), 0o755); err != nil { |
| + // It stays up until the stop: its exit removes its own child. |
| + if err := os.WriteFile(v2, []byte("#!/bin/sh\nexec sleep 30\n"), 0o755); err != nil { |
| t.Fatal(err) |
| } |
| if err := os.WriteFile(filepath.Join(b.DataDir, host+"-sidecar.env"), []byte("X=1\n"), 0o600); err != nil { |
| @@ -585,13 +588,16 @@ func TestStartSidecarGenKeepsTheRoleSidecarChild(t *testing.T) { |
| if err != nil { |
| t.Fatal(err) |
| } |
| - _ = p.Stop(context.Background()) |
| + // Checked before the stop: the exit's cleanup removes the empty child |
| + // (with clone3 placement nothing is written into it to keep it). |
| if fi, err := os.Stat(filepath.Join(parent, "role-sidecar")); err != nil || !fi.IsDir() { |
| + _ = p.Stop(context.Background()) |
| t.Fatalf("no role-sidecar child: %v", err) |
| } |
| if _, err := os.Stat(filepath.Join(parent, "role-sidecar", "memory.max")); !os.IsNotExist(err) { |
| t.Errorf("the sidecar's child got a limit of its own: %v", err) |
| } |
| + _ = p.Stop(context.Background()) |
| if got, err := os.ReadFile(filepath.Join(parent, "memory.oom.group")); err != nil || string(got) != "0" { |
| t.Errorf("parent memory.oom.group = %q, %v; want 0", got, err) |
| } |
diff --git a/internal/cgroup/cgroup.go b/internal/cgroup/cgroup.go| index 55fad4bb8..50572d945 100644 |
| --- a/internal/cgroup/cgroup.go |
| +++ b/internal/cgroup/cgroup.go |
| @@ -11,7 +11,10 @@ |
| // delegated subtree, never privilege. When delegation is absent (old unit |
| // file, a test, darwin) Setup returns a nil *Manager and every method on it |
| // degrades to today's behavior — fail toward absent readings, never toward |
| -// a broken spawn path. |
| +// a broken spawn path. The generation layout (StartChildGated, armed only |
| +// under CARLOS_SWAP) is the exception once a manager exists: a generation |
| +// whose limits cannot be applied is refused (ErrLimitNotApplied), never run |
| +// uncapped. |
| package cgroup |
| import ( |
| @@ -376,7 +379,26 @@ func (m *Manager) StartGated(host string, limit Limit, build func() *exec.Cmd, b |
| if err := m.freshen(host, m.hostDir(host)); err != nil { |
| return nil, err |
| } |
| - return m.startIn(host, m.hostDir(host), limit, build, before) |
| + return m.startIn(host, m.hostDir(host), limit, false, build, before) |
| +} |
| + |
| +// FakePlacementForTest is for another package's tests over a fake cgroupfs: |
| +// a temp directory is no cgroup, so the clone3 CLONE_INTO_CGROUP start |
| +// answers EBADF there, and a generation start (StartChildGated) is never |
| +// retried by start-and-move. Until t's cleanup, every start runs without |
| +// the cgroup fd, as if the kernel had placed it. t is a *testing.T; nothing |
| +// outside a test calls it. |
| +func FakePlacementForTest(t interface { |
| + Helper() |
| + Cleanup(func()) |
| +}) { |
| + t.Helper() |
| + orig := startCmd |
| + startCmd = func(cmd *exec.Cmd) error { |
| + clearInto(cmd) |
| + return orig(cmd) |
| + } |
| + t.Cleanup(func() { startCmd = orig }) |
| } |
| // startGated starts cmd once before, when set, has agreed. refused reports |
| @@ -404,15 +426,42 @@ func startPlain(build func() *exec.Cmd, before func() error) (*exec.Cmd, error) |
| // startIn is Start against an explicit cgroup directory: the host's own |
| // (Start) or one of its children (StartChild). The clone3 attempt, the |
| // fallback and its latch are Start's, unchanged; before is StartGated's. |
| -func (m *Manager) startIn(host, dir string, limit Limit, build func() *exec.Cmd, before func() error) (*exec.Cmd, error) { |
| +// |
| +// required is the generation layout's (StartChildGated, Task 7): every |
| +// limit must be in place before the process exists. With it, a child dir |
| +// that cannot be made or a limit write that fails refuses the start with |
| +// ErrLimitNotApplied, and the process is only ever placed by clone3 |
| +// CLONE_INTO_CGROUP: a latched fallback, a platform without clone3 |
| +// placement, or a dir that cannot be opened for the clone refuses before |
| +// build() runs, and a clone3 start that fails is not retried by |
| +// start-and-move. Without it (StartGated, the flat start) every one of |
| +// those stays best-effort, exactly as before. |
| +func (m *Manager) startIn(host, dir string, limit Limit, required bool, build func() *exec.Cmd, before func() error) (*exec.Cmd, error) { |
| + if required && (!intoCgroupSupported || m.fallback.Load()) { |
| + m.logf("cgroup: %s: clone3-into-cgroup is unavailable here, and a generation is never started then moved — generation start refused", host) |
| + return nil, fmt.Errorf("%w: %s: clone3-into-cgroup unavailable", ErrLimitNotApplied, host) |
| + } |
| if err := os.MkdirAll(dir, 0o755); err != nil { |
| + if required { |
| + m.logf("cgroup: %s: create cgroup: %v — generation start refused", host, err) |
| + return nil, fmt.Errorf("%w: %s: create %s: %w", ErrLimitNotApplied, host, filepath.Base(dir), err) |
| + } |
| m.logf("cgroup: %s: create cgroup: %v — spawning unaccounted", host, err) |
| return startPlain(build, before) |
| } |
| - m.applyLimit(host, dir, limit) |
| + if err := m.applyLimit(host, dir, limit); err != nil && required { |
| + _ = os.Remove(dir) |
| + m.logf("cgroup: %s: %s: generation start refused", host, filepath.Base(dir)) |
| + return nil, fmt.Errorf("%w: %s: %w", ErrLimitNotApplied, filepath.Base(dir), err) |
| + } |
| if intoCgroupSupported && !m.fallback.Load() { |
| f, err := os.Open(dir) |
| if err != nil { |
| + if required { |
| + _ = os.Remove(dir) |
| + m.logf("cgroup: %s: open cgroup dir: %v — generation start refused", host, err) |
| + return nil, fmt.Errorf("%w: %s: open %s for clone3: %w", ErrLimitNotApplied, host, filepath.Base(dir), err) |
| + } |
| m.logf("cgroup: %s: open cgroup dir: %v — using PID-move fallback for this spawn", host, err) |
| cmd, _, err := m.startAndMoveIn(host, dir, build, before) |
| return cmd, err |
| @@ -432,6 +481,19 @@ func (m *Manager) startIn(host, dir string, limit Limit, build func() *exec.Cmd, |
| return cmd, nil |
| } |
| firstErr := err |
| + if required { |
| + // Never start-and-move a generation: no retry, so nothing proves |
| + // the cgroup was the cause, and nothing latches. A failure that |
| + // is the placement's (placementFailed) is the box's condition; |
| + // any other is the start's own, as for a flat start whose retry |
| + // failed too. |
| + _ = os.Remove(dir) |
| + if placementFailed(firstErr) { |
| + m.logf("cgroup: %s: clone3-into-cgroup failed (%v) — generation start refused, not retried by PID move", host, firstErr) |
| + return nil, fmt.Errorf("%w: %s: clone3-into-cgroup: %w", ErrLimitNotApplied, host, firstErr) |
| + } |
| + return nil, firstErr |
| + } |
| retried, refused, retryErr := m.startAndMoveIn(host, dir, build, before) |
| if refused { |
| // The hook said no between the failed clone3 start and its |
| @@ -459,6 +521,8 @@ func (m *Manager) startIn(host, dir string, limit Limit, build func() *exec.Cmd, |
| // applyLimit writes limit into dir. Best-effort in the package's usual |
| // sense: a refused write (a kernel without the memory controller here, a |
| // control file that isn't ours) is one logged line and an uncapped child. |
| +// It returns the first write that failed, for a caller that must not start |
| +// without the whole limit (startIn's required); the flat start ignores it. |
| // Written BEFORE the child is in the cgroup, so the cap holds from its |
| // first page. memory.max first: if that one fails, high is pointless. |
| // |
| @@ -471,16 +535,18 @@ func (m *Manager) startIn(host, dir string, limit Limit, build func() *exec.Cmd, |
| // memory.oom.group=1 makes an OOM kill inside this cgroup take the whole |
| // family: a hub whose children outlive it would otherwise leave orphans |
| // holding the socket while the activator respawns a fresh hub beside them. |
| -func (m *Manager) applyLimit(host, dir string, limit Limit) { |
| +func (m *Manager) applyLimit(host, dir string, limit Limit) error { |
| if limit.MemoryMax <= 0 { |
| - return |
| + return nil |
| } |
| if err := os.WriteFile(filepath.Join(dir, "memory.max"), []byte(strconv.FormatInt(limit.MemoryMax, 10)), 0o644); err != nil { |
| m.logf("cgroup: %s: memory.max: %v — child runs uncapped", host, err) |
| - return |
| + return fmt.Errorf("memory.max: %w", err) |
| } |
| + var first error |
| if err := os.WriteFile(filepath.Join(dir, "memory.high"), []byte(strconv.FormatInt(HighOf(limit.MemoryMax), 10)), 0o644); err != nil { |
| m.logf("cgroup: %s: memory.high: %v — child is capped but not throttled first", host, err) |
| + first = fmt.Errorf("memory.high: %w", err) |
| } |
| // Swap is bounded too, or the cap isn't one: with zram on the box a |
| // cgroup at memory.max simply pushes its anonymous pages into |
| @@ -490,11 +556,18 @@ func (m *Manager) applyLimit(host, dir string, limit Limit) { |
| // gradient), then the kill is real. |
| if err := os.WriteFile(filepath.Join(dir, "memory.swap.max"), []byte(strconv.FormatInt(limit.MemoryMax, 10)), 0o644); err != nil { |
| m.logf("cgroup: %s: memory.swap.max: %v — the cap can be exceeded into swap", host, err) |
| + if first == nil { |
| + first = fmt.Errorf("memory.swap.max: %w", err) |
| + } |
| } |
| if err := os.WriteFile(filepath.Join(dir, "memory.oom.group"), []byte("1"), 0o644); err != nil { |
| m.logf("cgroup: %s: memory.oom.group: %v — an OOM kill here takes one process, not the family", host, err) |
| + if first == nil { |
| + first = fmt.Errorf("memory.oom.group: %w", err) |
| + } |
| } |
| m.logf("cgroup: %s: memory.max=%dM memory.high=%dM", host, limit.MemoryMax>>20, HighOf(limit.MemoryMax)>>20) |
| + return first |
| } |
| func (m *Manager) startAndMove(host string, build func() *exec.Cmd) (*exec.Cmd, error) { |
diff --git a/internal/cgroup/cgroup_linux.go b/internal/cgroup/cgroup_linux.go| index 5b362aab9..5448d97d9 100644 |
| --- a/internal/cgroup/cgroup_linux.go |
| +++ b/internal/cgroup/cgroup_linux.go |
| @@ -36,3 +36,24 @@ func setInto(cmd *exec.Cmd, fd int) { |
| func shouldLatch(err error) bool { |
| return errors.Is(err, syscall.EPERM) || errors.Is(err, syscall.ENOSYS) || errors.Is(err, syscall.EINVAL) |
| } |
| + |
| +// placementFailed reports whether a failed clone3 CLONE_INTO_CGROUP start |
| +// failed on the placement rather than on the command: the mechanism refused |
| +// (shouldLatch), or the kernel would not place into the cgroup the fd names |
| +// (EBADF: not a cgroup; ENODEV: removed under the open fd; EBUSY: a domain |
| +// controller enabled in it; EOPNOTSUPP: a domain-invalid cgroup). An exec |
| +// failure (ENOENT, EACCES, ENOEXEC, ...) is the command's own. A generation |
| +// start is never retried by start-and-move, so this is the only evidence |
| +// of which it was (Task 7). |
| +func placementFailed(err error) bool { |
| + return shouldLatch(err) || errors.Is(err, syscall.EBADF) || errors.Is(err, syscall.ENODEV) || |
| + errors.Is(err, syscall.EBUSY) || errors.Is(err, syscall.EOPNOTSUPP) |
| +} |
| + |
| +// clearInto undoes setInto: the start runs without the cgroup fd. |
| +func clearInto(cmd *exec.Cmd) { |
| + if cmd.SysProcAttr != nil { |
| + cmd.SysProcAttr.UseCgroupFD = false |
| + cmd.SysProcAttr.CgroupFD = 0 |
| + } |
| +} |
diff --git a/internal/cgroup/cgroup_linux_test.go b/internal/cgroup/cgroup_linux_test.go| index 19cd894ec..e0e769d38 100644 |
| --- a/internal/cgroup/cgroup_linux_test.go |
| +++ b/internal/cgroup/cgroup_linux_test.go |
| @@ -242,3 +242,82 @@ func TestStartGatedRefusedBeforeTheFirstStart(t *testing.T) { |
| t.Fatalf("host dir after a refused start: %v, want it removed", err) |
| } |
| } |
| + |
| +// Task 7, G: a generation start whose clone3 placement fails is never |
| +// retried by start-and-move. A failure that is the placement's (the |
| +// mechanism gone: EPERM, ENOSYS, EINVAL; or a cgroup fd the kernel would |
| +// not place into: EBADF, ENODEV, EBUSY, EOPNOTSUPP) is ErrLimitNotApplied, |
| +// the box's condition. Any other failure (a missing binary) is the start's |
| +// own error. Neither latches (no retry proved the cgroup was the cause), |
| +// and nothing is moved by PID. |
| +func TestAGenerationStartIsNeverRetriedByPIDMove(t *testing.T) { |
| + const host = "h.example.com" |
| + for name, tc := range map[string]struct { |
| + fail error |
| + want error |
| + }{ |
| + "clone3 refused": {syscall.EPERM, ErrLimitNotApplied}, |
| + "clone3 not understood": {syscall.EINVAL, ErrLimitNotApplied}, |
| + "not a cgroup": {syscall.EBADF, ErrLimitNotApplied}, |
| + "cgroup removed": {syscall.ENODEV, ErrLimitNotApplied}, |
| + "a spawn failure": {syscall.ENOENT, syscall.ENOENT}, |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + m, parent := genManager(t, host) |
| + calls := fakeStart(t, func(cmd *exec.Cmd) error { |
| + if cmd.SysProcAttr != nil && cmd.SysProcAttr.UseCgroupFD { |
| + return &os.PathError{Op: "fork/exec", Path: cmd.Path, Err: tc.fail} |
| + } |
| + return nil |
| + }) |
| + cmd, err := m.StartChild(host, GenChild(1), TierDefault, TierDefault, buildTrue) |
| + if !errors.Is(err, tc.want) || cmd != nil { |
| + t.Fatalf("StartChild = %v, %v: want %v and no command", cmd, err, tc.want) |
| + } |
| + if tc.want != ErrLimitNotApplied && errors.Is(err, ErrLimitNotApplied) { |
| + t.Errorf("a spawn failure read as a limit failure: %v", err) |
| + } |
| + if len(*calls) != 1 { |
| + t.Fatalf("started %d times, want the one clone3 attempt", len(*calls)) |
| + } |
| + if m.fallback.Load() { |
| + t.Error("a generation start latched the PID-move fallback") |
| + } |
| + if _, err := os.Stat(filepath.Join(parent, "g1", "cgroup.procs")); !os.IsNotExist(err) { |
| + t.Errorf("a PID was moved into g1: %v", err) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// FakePlacementForTest lets another package's fixture (a temp directory, |
| +// where clone3 CLONE_INTO_CGROUP answers EBADF) start a generation: the |
| +// clone3 start runs without the cgroup fd, as if placed, and is restored at |
| +// the test's cleanup. |
| +func TestFakePlacementForTestStartsWithoutTheFdAndRestores(t *testing.T) { |
| + const host = "h.example.com" |
| + m, _ := genManager(t, host) |
| + t.Run("faked", func(t *testing.T) { |
| + var seen []bool |
| + orig := startCmd |
| + startCmd = func(cmd *exec.Cmd) error { |
| + seen = append(seen, cmd.SysProcAttr != nil && cmd.SysProcAttr.UseCgroupFD) |
| + return syscall.EBADF |
| + } |
| + t.Cleanup(func() { startCmd = orig }) |
| + FakePlacementForTest(t) |
| + if _, err := m.StartChild(host, GenChild(1), TierDefault, TierDefault, buildTrue); !errors.Is(err, syscall.EBADF) { |
| + t.Fatalf("StartChild = %v, want the seam's own answer", err) |
| + } |
| + if len(seen) != 1 || seen[0] { |
| + t.Fatalf("starts with the cgroup fd = %v, want one start without it", seen) |
| + } |
| + }) |
| + calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| + if _, err := m.StartChild(host, GenChild(2), TierDefault, TierDefault, buildTrue); err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(*calls) != 1 || !(*calls)[0].SysProcAttr.UseCgroupFD { |
| + t.Fatal("the fake placement outlived its test") |
| + } |
| +} |
diff --git a/internal/cgroup/cgroup_other.go b/internal/cgroup/cgroup_other.go| index 04e8dbcbb..2317b0640 100644 |
| --- a/internal/cgroup/cgroup_other.go |
| +++ b/internal/cgroup/cgroup_other.go |
| @@ -12,3 +12,7 @@ const intoCgroupSupported = false |
| func setInto(cmd *exec.Cmd, fd int) {} |
| func shouldLatch(error) bool { return false } |
| + |
| +func placementFailed(error) bool { return false } |
| + |
| +func clearInto(*exec.Cmd) {} |
diff --git a/internal/cgroup/gen.go b/internal/cgroup/gen.go| index 69379ccf4..1d1c81a76 100644 |
| --- a/internal/cgroup/gen.go |
| +++ b/internal/cgroup/gen.go |
| @@ -9,6 +9,7 @@ import ( |
| "slices" |
| "strconv" |
| "strings" |
| + "syscall" |
| "github.com/carlosframework/platform/internal/controlname" |
| ) |
| @@ -101,9 +102,13 @@ func enableChildren(dir string) error { |
| if !strings.Contains(want, "+memory") { |
| return errors.New("no memory controller to delegate") |
| } |
| - return os.WriteFile(filepath.Join(dir, "cgroup.subtree_control"), []byte(want), 0o644) |
| + return writeSubtreeControl(filepath.Join(dir, "cgroup.subtree_control"), []byte(want)) |
| } |
| +// writeSubtreeControl is enableChildren's write, a seam so a test can |
| +// answer it as the kernel would (EBUSY for a parent a process landed in). |
| +var writeSubtreeControl = func(path string, b []byte) error { return os.WriteFile(path, b, 0o644) } |
| + |
| // parentPopulated reads dir's cgroup.events. An absent file means nothing |
| // has ever run there: not populated. A file that cannot be read counts as |
| // populated, so a raised limit is never lowered on a guess. |
| @@ -128,6 +133,36 @@ func parentPopulated(dir string) bool { |
| // is exec'd, and it is no fault of the build. |
| var ErrLegacyParentPopulated = errors.New("cgroup: the host's parent cgroup holds a process directly") |
| +// ErrLimitNotApplied is a generation start refused because a limit it would |
| +// run under could not be put in place: the parent could not be made, its |
| +// controllers could not be delegated (or a live parent does not delegate |
| +// memory), the parent's backstop could not be written, the generation's own |
| +// child could not be made or one of its four limit writes failed, or the |
| +// clone3 CLONE_INTO_CGROUP placement is unavailable, so the process could |
| +// only be started outside its cgroup and moved after. The flat start |
| +// (StartGated) stays best-effort and never returns it. Nothing is exec'd, |
| +// and it is no fault of the build: like ErrLegacyParentPopulated, the |
| +// condition is the box's (Task 7, zdd flag readiness round 2). |
| +var ErrLimitNotApplied = errors.New("cgroup: a generation's limits could not be applied") |
| + |
| +// delegatesMemory reports whether dir hands the memory controller to its |
| +// children: its cgroup.subtree_control lists memory. The kernel reads it |
| +// back without the "+" this package writes; either spelling counts. An |
| +// absent or unreadable file is not known to delegate, so it is refused as |
| +// one that does not. |
| +func delegatesMemory(dir string) error { |
| + b, err := os.ReadFile(filepath.Join(dir, "cgroup.subtree_control")) |
| + if err != nil { |
| + return fmt.Errorf("%w: cannot read the parent's cgroup.subtree_control: %w", ErrLimitNotApplied, err) |
| + } |
| + for _, f := range strings.Fields(string(b)) { |
| + if strings.TrimPrefix(f, "+") == "memory" { |
| + return nil |
| + } |
| + } |
| + return fmt.Errorf("%w: the parent does not delegate memory (cgroup.subtree_control %q)", ErrLimitNotApplied, strings.TrimSpace(string(b))) |
| +} |
| + |
| // parentProcs counts the processes dir holds directly: its own cgroup.procs, |
| // not cgroup.events, whose "populated" also counts every descendant. An |
| // absent file is a directory nothing has run in. A file that cannot be read |
| @@ -149,17 +184,21 @@ func parentProcs(dir string) (int, error) { |
| // the parent one. Then, when nothing is alive under it (a cold wake), it |
| // delegates the controllers and writes the tier with memory.oom.group=0. With |
| // a live generation under it, the parent is left as it is, because a swap |
| -// raised it on purpose. The refusal is the one failure that is not |
| -// best-effort: the rest of the accounting path logs a line and runs children |
| -// without their limits, but a refusal-free start here would run a generation |
| -// uncapped with nothing to say so. The error is ErrLegacyParentPopulated, |
| -// and an unreadable cgroup.procs is refused as the same (unknown is not |
| -// empty). |
| +// raised it on purpose, once it is shown to delegate memory to its children. |
| +// |
| +// Nothing here is best-effort (Task 7): a generation started past any of |
| +// these failures would run without a limit, with nothing to say so. A |
| +// populated parent is ErrLegacyParentPopulated, and an unreadable |
| +// cgroup.procs is refused as the same (unknown is not empty), as is a |
| +// subtree_control write the kernel answers EBUSY (a process landed in the |
| +// parent after the check). A parent that cannot be made, a live parent that |
| +// does not delegate memory, controllers that cannot be delegated, and a |
| +// parent backstop that cannot be written are ErrLimitNotApplied. |
| func (m *Manager) ensureParent(host string, tier Limit) error { |
| dir := m.hostDir(host) |
| if err := os.MkdirAll(dir, 0o755); err != nil { |
| - m.logf("cgroup: %s: create parent cgroup: %v (children run unaccounted)", host, err) |
| - return nil |
| + m.logf("cgroup: %s: create parent cgroup: %v (generation start refused)", host, err) |
| + return fmt.Errorf("%w: %s: create parent: %w", ErrLimitNotApplied, host, err) |
| } |
| n, err := parentProcs(dir) |
| if err != nil { |
| @@ -171,15 +210,24 @@ func (m *Manager) ensureParent(host string, tier Limit) error { |
| return fmt.Errorf("%w: %s holds %d process(es)", ErrLegacyParentPopulated, host, n) |
| } |
| if parentPopulated(dir) { |
| + if err := delegatesMemory(dir); err != nil { |
| + m.logf("cgroup: %s: generation start refused: %v", host, err) |
| + return fmt.Errorf("%s: %w", host, err) |
| + } |
| return nil |
| } |
| if err := enableChildren(dir); err != nil { |
| - m.logf("cgroup: %s: delegate controllers to generations: %v (children run uncapped)", host, err) |
| - return nil |
| + if errors.Is(err, syscall.EBUSY) { |
| + m.logf("cgroup: %s: generation start refused: delegate controllers: %v (a process holds the parent directly)", host, err) |
| + return fmt.Errorf("%w: %s: delegate controllers: %w", ErrLegacyParentPopulated, host, err) |
| + } |
| + m.logf("cgroup: %s: delegate controllers to generations: %v (generation start refused)", host, err) |
| + return fmt.Errorf("%w: %s: delegate controllers: %w", ErrLimitNotApplied, host, err) |
| } |
| if tier.MemoryMax > 0 { |
| if err := m.SetParentLimit(host, tier); err != nil { |
| - m.logf("cgroup: %s: parent limit: %v", host, err) |
| + m.logf("cgroup: %s: parent limit: %v (generation start refused)", host, err) |
| + return fmt.Errorf("%w: %w", ErrLimitNotApplied, err) |
| } |
| } |
| return nil |
| @@ -189,7 +237,10 @@ func (m *Manager) ensureParent(host string, tier Limit) error { |
| // carlos-exec@<host>/<child>. own is the child's own limit: the tier for a |
| // generation, zero for a role child, which then lives under the parent's |
| // limit alone. parent is the tier the parent takes when this spawn finds |
| -// nothing alive under it. Nil-safe and best-effort like Start. |
| +// nothing alive under it. Nil-safe like Start, but not best-effort: a |
| +// generation whose limits cannot all be applied is refused with |
| +// ErrLimitNotApplied before anything is built or exec'd (ensureParent, |
| +// startIn), and it is never started outside its cgroup and moved by PID. |
| func (m *Manager) StartChild(host, child string, parent, own Limit, build func() *exec.Cmd) (*exec.Cmd, error) { |
| return m.StartChildGated(host, child, parent, own, build, nil) |
| } |
| @@ -217,7 +268,7 @@ func (m *Manager) StartChildGated(host, child string, parent, own Limit, build f |
| if err := m.ensureParent(host, parent); err != nil { |
| return nil, err |
| } |
| - return m.startIn(host, filepath.Join(m.hostDir(host), child), own, build, before) |
| + return m.startIn(host, filepath.Join(m.hostDir(host), child), own, true, build, before) |
| } |
| // SetParentLimit writes l onto host's parent: memory.max, memory.high |
diff --git a/internal/cgroup/gen_kernel_linux_test.go b/internal/cgroup/gen_kernel_linux_test.go| index d6fc09c80..0881c68f2 100644 |
| --- a/internal/cgroup/gen_kernel_linux_test.go |
| +++ b/internal/cgroup/gen_kernel_linux_test.go |
| @@ -305,9 +305,10 @@ func TestGenOOMKernel(t *testing.T) { |
| if hogPipeErr != nil { |
| t.Fatalf("hog stdin: %v", hogPipeErr) |
| } |
| - // StartChild is best-effort: a refused limit write starts the child |
| - // uncapped. The hog waits on stdin, so prove it is confined first. Any |
| - // failure from here kills it: the cleanup registered above runs. |
| + // StartChild refuses a generation whose limits were not written (Task 7), |
| + // but prove the confinement here anyway, on the real kernel, before the |
| + // hog runs: it waits on stdin. Any failure from here kills it: the |
| + // cleanup registered above runs. |
| g2 := filepath.Join(parent, GenChild(2)) |
| wantG2 := filepath.Join(strings.TrimPrefix(m.Root(), "/sys/fs/cgroup/"), ExecName(host), GenChild(2)) |
| if got, err := os.ReadFile(filepath.Join(g2, "memory.max")); err != nil || strings.TrimSpace(string(got)) != "67108864" { |
diff --git a/internal/cgroup/gen_test.go b/internal/cgroup/gen_test.go| index 3361e47a9..e8131e759 100644 |
| --- a/internal/cgroup/gen_test.go |
| +++ b/internal/cgroup/gen_test.go |
| @@ -6,8 +6,10 @@ import ( |
| "os" |
| "os/exec" |
| "path/filepath" |
| + "slices" |
| "strconv" |
| "strings" |
| + "syscall" |
| "testing" |
| ) |
| @@ -135,6 +137,7 @@ func TestStartChildLeavesALiveParentAlone(t *testing.T) { |
| m, parent := genManager(t, host) |
| fakeStart(t, func(*exec.Cmd) error { return nil }) |
| writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 1\nfrozen 0\n") |
| + writeFixture(t, filepath.Join(parent, "cgroup.subtree_control"), "cpu memory io pids\n") |
| writeFixture(t, filepath.Join(parent, "memory.max"), "1073741824") |
| if _, err := m.StartChild(host, GenChild(2), TierDefault, TierDefault, buildTrue); err != nil { |
| t.Fatal(err) |
| @@ -414,6 +417,7 @@ func TestAnEmptyParentWithLiveGenerationsStillStartsASuccessor(t *testing.T) { |
| } |
| writeFixture(t, filepath.Join(parent, "g1", "cgroup.procs"), "5151\n") |
| writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 1\nfrozen 0\n") |
| + writeFixture(t, filepath.Join(parent, "cgroup.subtree_control"), "cpu memory io pids\n") |
| writeFixture(t, filepath.Join(parent, "memory.max"), "1073741824") |
| calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| if _, err := m.StartChild(host, GenChild(2), TierDefault, TierDefault, buildTrue); err != nil { |
| @@ -447,3 +451,263 @@ func TestAnUnreadableParentProcsFileIsRefusedAndAnAbsentOneIsNot(t *testing.T) { |
| t.Fatalf("StartChild over an unreadable cgroup.procs = %v, want ErrLegacyParentPopulated", err) |
| } |
| } |
| + |
| +// Task 7 (zdd flag readiness round 2): a generation whose limits cannot be |
| +// applied does not start. Each case below is one of the branches that used |
| +// to log a line and run the generation uncapped (recon C, A to G). Each |
| +// refuses with ErrLimitNotApplied before anything is exec'd (the build |
| +// function is never even called) and leaves no child cgroup behind. |
| + |
| +// refusedChild runs StartChild for g1 at the default tier and checks the |
| +// refusal: ErrLimitNotApplied (or want), no command, nothing started, and |
| +// the command never built. |
| +func refusedChild(t *testing.T, m *Manager, host, child string, own Limit, want error) error { |
| + t.Helper() |
| + calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| + built := false |
| + cmd, err := m.StartChild(host, child, TierDefault, own, func() *exec.Cmd { built = true; return buildTrue() }) |
| + if !errors.Is(err, want) || cmd != nil { |
| + t.Fatalf("StartChild(%s) = %v, %v: want %v and no command", child, cmd, err, want) |
| + } |
| + if len(*calls) != 0 || built { |
| + t.Fatalf("StartChild(%s): built %v, started %d times; want neither", child, built, len(*calls)) |
| + } |
| + return err |
| +} |
| + |
| +// noChildBut is noGenerationChild for a fixture that plants a directory |
| +// under the parent (a control file made a directory so its write fails): |
| +// the planted ones are the only directories allowed. |
| +func noChildBut(t *testing.T, parent string, planted ...string) { |
| + t.Helper() |
| + ents, err := os.ReadDir(parent) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, e := range ents { |
| + if e.IsDir() && !slices.Contains(planted, e.Name()) { |
| + t.Errorf("the refused start created %s under the parent", e.Name()) |
| + } |
| + } |
| +} |
| + |
| +// A: the parent cgroup cannot be made. It used to start the generation in |
| +// the agent's own cgroup, unaccounted and uncapped. The flat start (flag |
| +// off) keeps its best-effort plain start. |
| +func TestAGenerationParentThatCannotBeMadeIsRefused(t *testing.T) { |
| + const host = "h.example.com" |
| + m, parent := genManager(t, host) |
| + if err := os.RemoveAll(parent); err != nil { |
| + t.Fatal(err) |
| + } |
| + writeFixture(t, parent, "a file where the parent goes") // MkdirAll: ENOTDIR |
| + refusedChild(t, m, host, GenChild(1), TierDefault, ErrLimitNotApplied) |
| + if fi, err := os.Stat(parent); err != nil || fi.IsDir() { |
| + t.Fatalf("the refused start replaced the parent: %v", err) |
| + } |
| + calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| + if cmd, err := m.Start(host, TierDefault, buildTrue); err != nil || cmd == nil || len(*calls) != 1 { |
| + t.Fatalf("flag-off Start = %v, %v after %d starts: want today's plain start", cmd, err, len(*calls)) |
| + } |
| +} |
| + |
| +// B: a parent with live generations is left as it is, but only when it |
| +// delegates memory to its children. Without memory in its subtree_control |
| +// (absent, unreadable, or listing other controllers) a new g<n> under it |
| +// would get no memory.max at all. |
| +func TestALiveParentThatDoesNotDelegateMemoryIsRefused(t *testing.T) { |
| + const host = "h.example.com" |
| + for name, subtree := range map[string]*string{ |
| + "absent": nil, |
| + "without memory": ptr("cpu pids\n"), |
| + "empty": ptr(""), |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + m, parent := genManager(t, host) |
| + writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 1\nfrozen 0\n") |
| + if subtree != nil { |
| + writeFixture(t, filepath.Join(parent, "cgroup.subtree_control"), *subtree) |
| + } |
| + refusedChild(t, m, host, GenChild(2), TierDefault, ErrLimitNotApplied) |
| + noGenerationChild(t, parent) |
| + }) |
| + } |
| + t.Run("unreadable", func(t *testing.T) { |
| + m, parent := genManager(t, host) |
| + writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 1\nfrozen 0\n") |
| + if err := os.Mkdir(filepath.Join(parent, "cgroup.subtree_control"), 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + refusedChild(t, m, host, GenChild(2), TierDefault, ErrLimitNotApplied) |
| + noChildBut(t, parent, "cgroup.subtree_control") |
| + }) |
| + // Both spellings delegate: the kernel reads back "memory", and this |
| + // package's own write (and its fixtures) store "+memory". |
| + for _, subtree := range []string{"cpu memory io pids\n", "+cpu +memory +io +pids"} { |
| + m, parent := genManager(t, host) |
| + writeFixture(t, filepath.Join(parent, "cgroup.events"), "populated 1\nfrozen 0\n") |
| + writeFixture(t, filepath.Join(parent, "cgroup.subtree_control"), subtree) |
| + calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| + if _, err := m.StartChild(host, GenChild(2), TierDefault, TierDefault, buildTrue); err != nil || len(*calls) != 1 { |
| + t.Fatalf("subtree_control %q: StartChild = %v after %d starts, want one start", subtree, err, len(*calls)) |
| + } |
| + } |
| +} |
| + |
| +func ptr(s string) *string { return &s } |
| + |
| +// C: the controllers cannot be delegated on a cold start. A parent with no |
| +// memory controller to hand on, or a subtree_control write that fails, is |
| +// ErrLimitNotApplied; a write the kernel answers EBUSY is the populated |
| +// parent race (a process landed in the parent after the cgroup.procs |
| +// check), so it is ErrLegacyParentPopulated. Nothing else is written: no |
| +// parent limit, no child. |
| +func TestAParentWhoseControllersCannotBeDelegatedIsRefused(t *testing.T) { |
| + const host = "h.example.com" |
| + t.Run("no memory controller", func(t *testing.T) { |
| + m, parent := genManager(t, host) |
| + writeFixture(t, filepath.Join(parent, "cgroup.controllers"), "cpu pids\n") |
| + refusedChild(t, m, host, GenChild(1), TierDefault, ErrLimitNotApplied) |
| + noGenerationChild(t, parent) |
| + if _, err := os.Stat(filepath.Join(parent, "memory.max")); !os.IsNotExist(err) { |
| + t.Errorf("the refused start wrote the parent's limit: %v", err) |
| + } |
| + }) |
| + for name, tc := range map[string]struct { |
| + write error |
| + want error |
| + }{ |
| + "the write is refused": {syscall.EACCES, ErrLimitNotApplied}, |
| + "a process landed in the parent": {syscall.EBUSY, ErrLegacyParentPopulated}, |
| + } { |
| + t.Run(name, func(t *testing.T) { |
| + m, parent := genManager(t, host) |
| + orig := writeSubtreeControl |
| + writeSubtreeControl = func(path string, _ []byte) error { |
| + return &os.PathError{Op: "write", Path: path, Err: tc.write} |
| + } |
| + t.Cleanup(func() { writeSubtreeControl = orig }) |
| + err := refusedChild(t, m, host, GenChild(1), TierDefault, tc.want) |
| + noGenerationChild(t, parent) |
| + if tc.want == ErrLegacyParentPopulated && errors.Is(err, ErrLimitNotApplied) { |
| + t.Error("the EBUSY race reads as a limit failure") |
| + } |
| + if _, err := os.Stat(filepath.Join(parent, "memory.max")); !os.IsNotExist(err) { |
| + t.Errorf("the refused start wrote the parent's limit: %v", err) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// D: the parent's backstop cannot be written. A role child (the legacy |
| +// sidecar, the seed) has no limit of its own, so without the parent's it |
| +// would have none at all. |
| +func TestAParentWhoseLimitCannotBeWrittenIsRefused(t *testing.T) { |
| + const host = "h.example.com" |
| + for _, file := range []string{"memory.max", "memory.oom.group"} { |
| + for _, child := range []string{GenChild(1), RoleChild("sidecar")} { |
| + m, parent := genManager(t, host) |
| + if err := os.Mkdir(filepath.Join(parent, file), 0o755); err != nil { // WriteFile: EISDIR |
| + t.Fatal(err) |
| + } |
| + own := Limit{} |
| + if child == GenChild(1) { |
| + own = TierDefault |
| + } |
| + refusedChild(t, m, host, child, own, ErrLimitNotApplied) |
| + noChildBut(t, parent, file) |
| + } |
| + } |
| +} |
| + |
| +// E: the generation's own child cgroup cannot be made. It used to start in |
| +// the agent's cgroup, unaccounted and uncapped. |
| +func TestAGenerationChildThatCannotBeMadeIsRefused(t *testing.T) { |
| + const host = "h.example.com" |
| + m, parent := genManager(t, host) |
| + writeFixture(t, filepath.Join(parent, "g1"), "a file where g1 goes") // MkdirAll: ENOTDIR |
| + refusedChild(t, m, host, GenChild(1), TierDefault, ErrLimitNotApplied) |
| + noGenerationChild(t, parent) |
| +} |
| + |
| +// F: one of the generation's four limit writes fails. All four are the |
| +// child's limits (spec §7.3), so any failure refuses the start, and the |
| +// process never lands in the child (no cgroup.procs write). The flat start |
| +// keeps logging and spawning (TestStartLimitWriteFailureStillSpawns). |
| +func TestAGenerationWhoseLimitsCannotBeWrittenIsRefused(t *testing.T) { |
| + const host = "h.example.com" |
| + for _, file := range []string{"memory.max", "memory.high", "memory.swap.max", "memory.oom.group"} { |
| + t.Run(file, func(t *testing.T) { |
| + m, parent := genManager(t, host) |
| + g1 := filepath.Join(parent, "g1") |
| + if err := os.MkdirAll(filepath.Join(g1, file), 0o755); err != nil { // WriteFile: EISDIR |
| + t.Fatal(err) |
| + } |
| + refusedChild(t, m, host, GenChild(1), TierDefault, ErrLimitNotApplied) |
| + if _, err := os.Stat(filepath.Join(g1, "cgroup.procs")); !os.IsNotExist(err) { |
| + t.Errorf("a process was placed in the refused child: %v", err) |
| + } |
| + noChildBut(t, parent, "g1") |
| + }) |
| + } |
| + // applyLimit reports the first failure; with every write landing it |
| + // answers nil. |
| + m, parent := genManager(t, host) |
| + dir := filepath.Join(parent, "g9") |
| + if err := os.Mkdir(dir, 0o755); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := m.applyLimit(host, dir, TierDefault); err != nil { |
| + t.Fatalf("applyLimit with every write landing = %v", err) |
| + } |
| + if err := m.applyLimit(host, dir, Limit{}); err != nil { |
| + t.Fatalf("applyLimit with no limit = %v", err) |
| + } |
| +} |
| + |
| +// G: with the clone3 CLONE_INTO_CGROUP placement unavailable, a generation |
| +// could only be started and then moved by PID: running, for that window, |
| +// in the agent's cgroup with no limit, and for good if the move fails. |
| +// A latched fallback refuses before anything is created or built. The flat |
| +// start keeps the PID-move fallback. |
| +func TestAGenerationIsNeverStartedThenMovedWhenTheFallbackIsLatched(t *testing.T) { |
| + const host = "h.example.com" |
| + m, parent := genManager(t, host) |
| + m.fallback.Store(true) |
| + for _, child := range []string{GenChild(1), RoleChild("seed")} { |
| + own := Limit{} |
| + if child == GenChild(1) { |
| + own = TierDefault |
| + } |
| + refusedChild(t, m, host, child, own, ErrLimitNotApplied) |
| + noGenerationChild(t, parent) |
| + } |
| + calls := fakeStart(t, func(*exec.Cmd) error { return nil }) |
| + if cmd, err := m.Start("flat.example.com", TierDefault, buildTrue); err != nil || cmd == nil || len(*calls) != 1 { |
| + t.Fatalf("flag-off Start with the latch = %v, %v after %d starts: want the PID-move start", cmd, err, len(*calls)) |
| + } |
| + if _, err := os.Stat(filepath.Join(m.hostDir("flat.example.com"), "cgroup.procs")); err != nil { |
| + t.Fatalf("flag-off Start did not move the PID: %v", err) |
| + } |
| +} |
| + |
| +// G: the child cgroup cannot be opened for the clone. A flat start would |
| +// fall back to start-and-move for this spawn; a generation start refuses |
| +// before it builds the command, and drops the child it made. |
| +func TestAGenerationWhoseCgroupCannotBeOpenedIsRefused(t *testing.T) { |
| + if !intoCgroupSupported { |
| + t.Skip("no clone3 placement on this platform; every generation start refuses") |
| + } |
| + if os.Geteuid() == 0 { |
| + t.Skip("root opens a mode-0 directory") |
| + } |
| + const host = "h.example.com" |
| + m, parent := genManager(t, host) |
| + seed := filepath.Join(parent, RoleChild("seed")) |
| + if err := os.Mkdir(seed, 0o000); err != nil { // open: EACCES |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { _ = os.Chmod(seed, 0o755) }) |
| + refusedChild(t, m, host, RoleChild("seed"), Limit{}, ErrLimitNotApplied) |
| + noGenerationChild(t, parent) |
| +} |
diff --git a/internal/cgroup/recreate_linux_test.go b/internal/cgroup/recreate_linux_test.go| index 598cd9b63..69707f6ee 100644 |
| --- a/internal/cgroup/recreate_linux_test.go |
| +++ b/internal/cgroup/recreate_linux_test.go |
| @@ -472,6 +472,7 @@ func TestARetainingSetupMakesEmptyLeftoversFresh(t *testing.T) { |
| // A generation parent whose live g3 survived, beside an empty sidecar |
| // child a KillSidecar left. |
| live := retainedTree(t, root, "live.example.com", "populated 1\n", "", map[string]string{"g3": "populated 1\n", "role-sidecar": "populated 0\n"}) |
| + writeFixture(t, filepath.Join(live, "cgroup.subtree_control"), "cpu memory io pids\n") |
| claim := retainedTree(t, root, "claim-0123abcd.local", "populated 0\n", "", nil) |
| for _, d := range []string{empty, busy, live, filepath.Join(live, "g3"), filepath.Join(live, "role-sidecar")} { |
| plantSentinel(t, d) |