| 1 | //go:build linux |
| 2 | |
| 3 | package activator |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "os" |
| 10 | "os/exec" |
| 11 | "path/filepath" |
| 12 | "slices" |
| 13 | "strconv" |
| 14 | "strings" |
| 15 | "syscall" |
| 16 | "testing" |
| 17 | "time" |
| 18 | |
| 19 | "github.com/carlosframework/platform/internal/cgroup" |
| 20 | "github.com/carlosframework/platform/internal/provision" |
| 21 | "github.com/carlosframework/platform/internal/registry" |
| 22 | ) |
| 23 | |
| 24 | func TestClaimSpawnKernelChild(t *testing.T) { |
| 25 | if os.Getenv("CARLOS_CLAIM_SPAWN_CHILD") != "1" { |
| 26 | t.Skip("synthetic contained child only") |
| 27 | } |
| 28 | time.Sleep(time.Minute) |
| 29 | } |
| 30 | |
| 31 | // TestClaimContainmentCleanupRespawn kills a cgroup and spawns into the |
| 32 | // same path again, on a real kernel. On the dev box (amadan #272: arm64, |
| 33 | // kernel 6.12.105, systemd 257) a clone3(CLONE_INTO_CGROUP) into a cgroup |
| 34 | // that took cgroup.kill is SIGKILLed at once, so each respawn below must |
| 35 | // land in a fresh cgroup and stay alive. The strict primitive still places |
| 36 | // with clone3; the Manager's starts (the worker's and the sidecar's) now |
| 37 | // place through the exec-shim, whose cgroup.procs write is not clone3's, |
| 38 | // and must still land in a fresh cgroup, limits written again, and stay |
| 39 | // up. Three sequences: the strict |
| 40 | // primitive (PrepareStrict, then a cleanup handle's kill), a worker's |
| 41 | // KillHost then its respawn into carlos-exec@<host>, and the legacy |
| 42 | // sidecar's KillSidecar then its respawn into role-sidecar. |
| 43 | func TestClaimContainmentCleanupRespawn(t *testing.T) { |
| 44 | if os.Getenv("CARLOS_CLAIM_SPAWN_KERNEL") != "1" { |
| 45 | t.Skip("requires a disposable delegated user service") |
| 46 | } |
| 47 | manager := cgroup.Setup(t.Logf) |
| 48 | if manager == nil { |
| 49 | t.Fatal("missing delegated cgroup") |
| 50 | } |
| 51 | dir := t.TempDir() |
| 52 | build := func() *exec.Cmd { |
| 53 | cmd := exec.Command(os.Args[0], "-test.run=^TestClaimSpawnKernelChild$", "-test.count=1") |
| 54 | cmd.Env = []string{"CARLOS_CLAIM_SPAWN_CHILD=1"} |
| 55 | cmd.Dir = dir |
| 56 | return cmd |
| 57 | } |
| 58 | t.Run("strict primitive", func(t *testing.T) { claimStrictRespawn(t, manager, build) }) |
| 59 | t.Run("KillHost then respawn", func(t *testing.T) { killHostRespawn(t, manager, build) }) |
| 60 | t.Run("KillSidecar then respawn", func(t *testing.T) { killSidecarRespawn(t, manager, build) }) |
| 61 | } |
| 62 | |
| 63 | func claimStrictRespawn(t *testing.T, manager *cgroup.Manager, build func() *exec.Cmd) { |
| 64 | const host = "claimspawn.example" |
| 65 | for attempt := range 2 { |
| 66 | // This tests the cgroup primitive with a synthetic test child. It |
| 67 | // grants no claim launch authority and is not a tenant-isolation test. |
| 68 | prepared, err := manager.PrepareStrict(host, cgroup.TierDefault) |
| 69 | if err != nil { |
| 70 | t.Fatal(err) |
| 71 | } |
| 72 | cmd := build() |
| 73 | err = prepared.Start(cmd) |
| 74 | if err != nil || cmd == nil || prepared == nil { |
| 75 | t.Fatalf("spawn %d: cmd=%v prepared=%v err=%v", attempt, cmd, prepared, err) |
| 76 | } |
| 77 | // The unit's KillMode covers fatal test exits. Normal cleanup below |
| 78 | // must release the original reservation without removing the group. |
| 79 | if err := prepared.Close(); !errors.Is(err, cgroup.ErrStrictRefused) { |
| 80 | t.Fatalf("populated close: %v", err) |
| 81 | } |
| 82 | cleanup, err := manager.OpenCleanup(host) |
| 83 | if err != nil { |
| 84 | t.Fatal(err) |
| 85 | } |
| 86 | ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) |
| 87 | err = cleanup.KillAndEmpty(ctx) |
| 88 | cancel() |
| 89 | if err != nil { |
| 90 | t.Fatal(err) |
| 91 | } |
| 92 | var exitErr *exec.ExitError |
| 93 | if err := cmd.Wait(); err != nil && !errors.As(err, &exitErr) { |
| 94 | t.Fatal(err) |
| 95 | } |
| 96 | if cmd.ProcessState == nil { |
| 97 | t.Fatal("child was not reaped") |
| 98 | } |
| 99 | if err := cleanup.Close(); err != nil { |
| 100 | t.Fatal(err) |
| 101 | } |
| 102 | if other, err := manager.PrepareStrict(host, cgroup.TierDefault); other != nil || !errors.Is(err, cgroup.ErrStrictState) { |
| 103 | t.Fatalf("cleanup close released original reservation: %v", err) |
| 104 | } |
| 105 | if populated, err := prepared.Populated(); err != nil || populated { |
| 106 | t.Fatalf("pinned group not proven empty: %v %v", populated, err) |
| 107 | } |
| 108 | if err := prepared.Close(); err != nil { |
| 109 | t.Fatal(err) |
| 110 | } |
| 111 | } |
| 112 | } |
| 113 | |
| 114 | // spawnStaysUp fails the test if cmd exits within a moment of its start: |
| 115 | // the #272 symptom is a child SIGKILLed as it lands. It returns the |
| 116 | // channel cmd.Wait's result arrives on. |
| 117 | func spawnStaysUp(t *testing.T, label string, cmd *exec.Cmd) <-chan error { |
| 118 | t.Helper() |
| 119 | done := make(chan error, 1) |
| 120 | go func() { done <- cmd.Wait() }() |
| 121 | select { |
| 122 | case err := <-done: |
| 123 | t.Fatalf("%s: the child died as it landed: %v", label, err) |
| 124 | case <-time.After(500 * time.Millisecond): |
| 125 | } |
| 126 | return done |
| 127 | } |
| 128 | |
| 129 | // reaped waits for a killed child's exit. |
| 130 | func reaped(t *testing.T, label string, done <-chan error) { |
| 131 | t.Helper() |
| 132 | select { |
| 133 | case <-done: |
| 134 | case <-time.After(5 * time.Second): |
| 135 | t.Fatalf("%s: the killed child was not reaped", label) |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | func cgroupInode(t *testing.T, path string) uint64 { |
| 140 | t.Helper() |
| 141 | var st syscall.Stat_t |
| 142 | if err := syscall.Lstat(path, &st); err != nil { |
| 143 | t.Fatalf("stat %s: %v", path, err) |
| 144 | } |
| 145 | return st.Ino |
| 146 | } |
| 147 | |
| 148 | // killHostRespawn is a worker's sequence: spawned through spawnInCgroup into |
| 149 | // carlos-exec@<host>, proven terminated by the real ExecBackend.KillHost, |
| 150 | // then respawned into the same path. Each respawn gets a fresh cgroup with |
| 151 | // the tier written again, and stays up. |
| 152 | func killHostRespawn(t *testing.T, manager *cgroup.Manager, build func() *exec.Cmd) { |
| 153 | const host = "killhost.claimspawn.example" |
| 154 | b := &ExecBackend{Cgroups: manager} |
| 155 | rt := registry.Route{Host: host} |
| 156 | dir := filepath.Join(manager.Root(), cgroup.ExecName(host)) |
| 157 | var last uint64 |
| 158 | for attempt := range 3 { |
| 159 | cmd, settle, err := b.spawnInCgroup(rt, build) |
| 160 | if err != nil { |
| 161 | t.Fatalf("spawn %d: %v", attempt, err) |
| 162 | } |
| 163 | done := spawnStaysUp(t, fmt.Sprintf("spawn %d", attempt), cmd) |
| 164 | ino := cgroupInode(t, dir) |
| 165 | if attempt > 0 && ino == last { |
| 166 | t.Fatalf("spawn %d reused the killed host cgroup", attempt) |
| 167 | } |
| 168 | last = ino |
| 169 | if got, err := os.ReadFile(filepath.Join(dir, "memory.max")); err != nil || strings.TrimSpace(string(got)) != strconv.FormatInt(cgroup.TierDefault.MemoryMax, 10) { |
| 170 | t.Fatalf("spawn %d: memory.max = %q, %v; want the tier", attempt, got, err) |
| 171 | } |
| 172 | ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) |
| 173 | err = b.KillHost(ctx, rt) |
| 174 | cancel() |
| 175 | if err != nil { |
| 176 | t.Fatalf("KillHost %d: %v", attempt, err) |
| 177 | } |
| 178 | reaped(t, fmt.Sprintf("spawn %d", attempt), done) |
| 179 | settle() |
| 180 | } |
| 181 | b.cleanupHost(host) |
| 182 | } |
| 183 | |
| 184 | // killSidecarRespawn is the legacy sidecar's sequence under containment: a |
| 185 | // flat row's sidecar spawned into carlos-exec@<host>/role-sidecar, proven |
| 186 | // gone by the real ExecBackend.KillSidecar, then respawned. Each respawn gets |
| 187 | // a fresh role-sidecar under the same parent, and stays up. A live g1 runs |
| 188 | // under the parent throughout (zdd readiness round 2, Task 7): a parent |
| 189 | // populated only through its children is no legacy parent, so no respawn |
| 190 | // is refused, the parent never holds a process directly, it keeps memory |
| 191 | // delegated, and g1 is untouched by every kill. |
| 192 | func killSidecarRespawn(t *testing.T, manager *cgroup.Manager, build func() *exec.Cmd) { |
| 193 | const host = "killsidecar.claimspawn.example" |
| 194 | b := &ExecBackend{Cgroups: manager, GenContainment: true, DataDir: t.TempDir(), OptDir: t.TempDir()} |
| 195 | rt := registry.Route{Host: host, Kind: "instance", DBPath: provision.DefaultDBPath(b.DataDir, host), |
| 196 | Socket: filepath.Join(provision.SocketDir, host+".sock")} |
| 197 | parent := filepath.Join(manager.Root(), cgroup.ExecName(host)) |
| 198 | child := cgroup.RoleChild("sidecar") |
| 199 | g1, _, err := b.spawnInCgroupAt(rt, cgroup.GenChild(1), true, build) |
| 200 | if err != nil || g1 == nil { |
| 201 | t.Fatalf("live g1: %v, %v", g1, err) |
| 202 | } |
| 203 | t.Cleanup(func() { _ = g1.Process.Kill() }) |
| 204 | g1Done := spawnStaysUp(t, "live g1", g1) |
| 205 | var last, parentIno uint64 |
| 206 | for attempt := range 3 { |
| 207 | cmd, settle, err := b.spawnInCgroupAt(rt, child, false, build) |
| 208 | if err != nil { |
| 209 | t.Fatalf("spawn %d beside a live g1: %v", attempt, err) |
| 210 | } |
| 211 | if got, err := os.ReadFile(filepath.Join(parent, "cgroup.procs")); err != nil || strings.TrimSpace(string(got)) != "" { |
| 212 | t.Fatalf("spawn %d: the parent holds processes directly: %q, %v", attempt, got, err) |
| 213 | } |
| 214 | if got, err := os.ReadFile(filepath.Join(parent, "cgroup.subtree_control")); err != nil || !slices.Contains(strings.Fields(string(got)), "memory") { |
| 215 | t.Fatalf("spawn %d: the parent's subtree_control is %q, %v; want memory delegated", attempt, got, err) |
| 216 | } |
| 217 | done := spawnStaysUp(t, fmt.Sprintf("sidecar spawn %d", attempt), cmd) |
| 218 | ino := cgroupInode(t, filepath.Join(parent, child)) |
| 219 | if attempt > 0 && ino == last { |
| 220 | t.Fatalf("sidecar spawn %d reused the killed role-sidecar", attempt) |
| 221 | } |
| 222 | if p := cgroupInode(t, parent); attempt > 0 && p != parentIno { |
| 223 | t.Fatalf("sidecar spawn %d replaced the parent", attempt) |
| 224 | } else { |
| 225 | parentIno = p |
| 226 | } |
| 227 | last = ino |
| 228 | ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) |
| 229 | err = b.KillSidecar(ctx, rt) |
| 230 | cancel() |
| 231 | if err != nil { |
| 232 | t.Fatalf("KillSidecar %d: %v", attempt, err) |
| 233 | } |
| 234 | reaped(t, fmt.Sprintf("sidecar spawn %d", attempt), done) |
| 235 | settle() |
| 236 | } |
| 237 | select { |
| 238 | case err := <-g1Done: |
| 239 | t.Fatalf("the live g1 died during the sidecar respawns: %v", err) |
| 240 | default: |
| 241 | } |
| 242 | ctx, cancel := context.WithTimeout(t.Context(), 3*time.Second) |
| 243 | err = b.KillGen(ctx, rt, 1) |
| 244 | cancel() |
| 245 | if err != nil { |
| 246 | t.Fatalf("KillGen 1: %v", err) |
| 247 | } |
| 248 | reaped(t, "live g1", g1Done) |
| 249 | manager.RemoveChild(host, child) |
| 250 | manager.CleanupHost(host) |
| 251 | } |
| 252 | |
amadan