| 1 | package activator |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "os" |
| 8 | "os/exec" |
| 9 | "path/filepath" |
| 10 | "strconv" |
| 11 | "strings" |
| 12 | "syscall" |
| 13 | "testing" |
| 14 | |
| 15 | "github.com/carlosframework/platform/internal/cgroup" |
| 16 | "github.com/carlosframework/platform/internal/instancectl" |
| 17 | "github.com/carlosframework/platform/internal/registry" |
| 18 | ) |
| 19 | |
| 20 | // fakeGenControl adds the generation verbs to fakeControl (unitproc_test.go). |
| 21 | type fakeGenControl struct{ *fakeControl } |
| 22 | |
| 23 | func (f fakeGenControl) StartGen(_ context.Context, host string, n uint64, max int64) (string, error) { |
| 24 | f.rec(fmt.Sprintf("startgen %s %d %d", host, n, max)) |
| 25 | if f.onStart != nil { |
| 26 | f.onStart() |
| 27 | } |
| 28 | return f.inv, f.startErr |
| 29 | } |
| 30 | func (f fakeGenControl) StartGenExe(ctx context.Context, host string, n uint64, max int64, _ instancectl.StagedExe) (string, error) { |
| 31 | return f.StartGen(ctx, host, n, max) |
| 32 | } |
| 33 | func (f fakeGenControl) StartSidecarExe(_ context.Context, host string, max int64, _ instancectl.StagedExe) (string, error) { |
| 34 | f.rec(fmt.Sprintf("startsidecarexe %s %d", host, max)) |
| 35 | return f.inv, f.startErr |
| 36 | } |
| 37 | func (f fakeGenControl) StopGen(_ context.Context, host string, n uint64, kill bool) error { |
| 38 | f.rec(fmt.Sprintf("stopgen %s %d kill=%v", host, n, kill)) |
| 39 | if f.stopHook != nil { |
| 40 | f.stopHook(host, instancectl.RoleRun) |
| 41 | } |
| 42 | return f.stopGenErr |
| 43 | } |
| 44 | func (f fakeGenControl) ResetGen(_ context.Context, host string, n uint64) error { |
| 45 | f.rec(fmt.Sprintf("resetgen %s %d", host, n)) |
| 46 | return nil |
| 47 | } |
| 48 | func (f fakeGenControl) Parent(_ context.Context, host string, max int64) error { |
| 49 | f.rec(fmt.Sprintf("parent %s %d", host, max)) |
| 50 | return nil |
| 51 | } |
| 52 | |
| 53 | func TestPlaceForIsSingleLevelWithContainmentOff(t *testing.T) { |
| 54 | b := testExecBackend(t) |
| 55 | for _, kind := range []spawnKind{spawnInstance, spawnSidecar, spawnSeed, spawnWorker} { |
| 56 | child, own, err := b.placeFor(kind, 7) |
| 57 | if child != "" || own || err != nil { |
| 58 | t.Errorf("kind %d with containment off: %q %v %v", kind, child, own, err) |
| 59 | } |
| 60 | } |
| 61 | } |
| 62 | |
| 63 | func TestPlaceForUnderContainment(t *testing.T) { |
| 64 | b := testExecBackend(t) |
| 65 | b.GenContainment = true |
| 66 | for _, tc := range []struct { |
| 67 | kind spawnKind |
| 68 | child string |
| 69 | own bool |
| 70 | }{ |
| 71 | {spawnInstance, "g7", true}, |
| 72 | {spawnSidecar, "role-sidecar", false}, |
| 73 | {spawnSeed, "role-seed", false}, |
| 74 | {spawnWorker, "", false}, |
| 75 | } { |
| 76 | child, own, err := b.placeFor(tc.kind, 7) |
| 77 | if child != tc.child || own != tc.own || err != nil { |
| 78 | t.Errorf("kind %d: %q %v %v, want %q %v", tc.kind, child, own, err, tc.child, tc.own) |
| 79 | } |
| 80 | } |
| 81 | if _, _, err := b.placeFor(spawnInstance, 0); err == nil { |
| 82 | t.Error("an instance spawn with generation 0 was placed") |
| 83 | } |
| 84 | } |
| 85 | |
| 86 | func delegatedBackend(t *testing.T, host string) (*ExecBackend, string) { |
| 87 | t.Helper() |
| 88 | root := t.TempDir() |
| 89 | for _, f := range []string{"cgroup.procs", "cgroup.subtree_control"} { |
| 90 | if err := os.WriteFile(filepath.Join(root, f), nil, 0o644); err != nil { |
| 91 | t.Fatal(err) |
| 92 | } |
| 93 | } |
| 94 | m, err := cgroup.SetupAt(root, t.Logf) |
| 95 | if err != nil { |
| 96 | t.Fatal(err) |
| 97 | } |
| 98 | parent := filepath.Join(root, cgroup.ExecName(host)) |
| 99 | if err := os.MkdirAll(parent, 0o755); err != nil { |
| 100 | t.Fatal(err) |
| 101 | } |
| 102 | if err := os.WriteFile(filepath.Join(parent, "cgroup.controllers"), []byte("cpu io memory pids\n"), 0o644); err != nil { |
| 103 | t.Fatal(err) |
| 104 | } |
| 105 | b := testExecBackend(t) |
| 106 | b.Cgroups = m |
| 107 | return b, parent |
| 108 | } |
| 109 | |
| 110 | func TestSpawnInCgroupAtPutsAGenerationUnderItsParent(t *testing.T) { |
| 111 | const host = "h.example.com" |
| 112 | b, parent := delegatedBackend(t, host) |
| 113 | cgroup.FakePlacementForTest(t) // a temp dir is no cgroupfs, and a generation is never moved by PID |
| 114 | b.GenContainment = true |
| 115 | cmd, _, err := b.spawnInCgroupAt(registry.Route{Host: host}, "g7", true, func() *exec.Cmd { return exec.Command("/bin/true") }) |
| 116 | if err != nil { |
| 117 | t.Fatal(err) |
| 118 | } |
| 119 | _ = cmd.Wait() |
| 120 | for file, want := range map[string]string{ |
| 121 | filepath.Join(parent, "g7", "memory.max"): "536870912", |
| 122 | filepath.Join(parent, "g7", "memory.oom.group"): "1", |
| 123 | filepath.Join(parent, "memory.oom.group"): "0", |
| 124 | } { |
| 125 | if got, err := os.ReadFile(file); err != nil || string(got) != want { |
| 126 | t.Errorf("%s = %q, %v; want %q", file, got, err, want) |
| 127 | } |
| 128 | } |
| 129 | } |
| 130 | |
| 131 | // Flag off: the spawn lands in the host cgroup itself, as today. |
| 132 | func TestSpawnInCgroupAtWithNoChildKeepsTheHostCgroup(t *testing.T) { |
| 133 | const host = "h.example.com" |
| 134 | b, parent := delegatedBackend(t, host) |
| 135 | cmd, _, err := b.spawnInCgroupAt(registry.Route{Host: host}, "", false, func() *exec.Cmd { return exec.Command("/bin/true") }) |
| 136 | if err != nil { |
| 137 | t.Fatal(err) |
| 138 | } |
| 139 | _ = cmd.Wait() |
| 140 | if got, err := os.ReadFile(filepath.Join(parent, "memory.oom.group")); err != nil || string(got) != "1" { |
| 141 | t.Errorf("host cgroup memory.oom.group = %q, %v; want 1", got, err) |
| 142 | } |
| 143 | entries, _ := os.ReadDir(parent) |
| 144 | for _, e := range entries { |
| 145 | if e.IsDir() { |
| 146 | t.Errorf("a flag-off spawn made child cgroup %s", e.Name()) |
| 147 | } |
| 148 | } |
| 149 | } |
| 150 | |
| 151 | func TestParentLimitRefusesWithContainmentOff(t *testing.T) { |
| 152 | b, _ := delegatedBackend(t, "h.example.com") |
| 153 | if err := b.ParentLimit(context.Background(), registry.Route{Host: "h.example.com"}, cgroup.TierDefault); !errors.Is(err, errNoGenContainment) { |
| 154 | t.Fatalf("ParentLimit with containment off: %v", err) |
| 155 | } |
| 156 | if err := b.KillGen(context.Background(), registry.Route{Host: "h.example.com"}, 7); !errors.Is(err, errNoGenContainment) { |
| 157 | t.Fatalf("KillGen with containment off: %v", err) |
| 158 | } |
| 159 | } |
| 160 | |
| 161 | func TestParentLimitFlatWritesTheParent(t *testing.T) { |
| 162 | const host = "h.example.com" |
| 163 | b, parent := delegatedBackend(t, host) |
| 164 | b.GenContainment = true |
| 165 | if err := b.ParentLimit(context.Background(), registry.Route{Host: host}, cgroup.OverlapLimit(cgroup.TierDefault, cgroup.TierDefault)); err != nil { |
| 166 | t.Fatal(err) |
| 167 | } |
| 168 | if got, err := os.ReadFile(filepath.Join(parent, "memory.max")); err != nil || string(got) != "1073741824" { |
| 169 | t.Errorf("parent memory.max = %q, %v", got, err) |
| 170 | } |
| 171 | } |
| 172 | |
| 173 | // No cgroup manager (no delegation): there is no parent to raise, so a |
| 174 | // flat ParentLimit refuses rather than let a swap admit an overlap with |
| 175 | // nothing raised. |
| 176 | func TestParentLimitFlatRefusesWithoutACgroupManager(t *testing.T) { |
| 177 | b := testExecBackend(t) |
| 178 | b.GenContainment = true |
| 179 | b.Cgroups = nil |
| 180 | if err := b.ParentLimit(context.Background(), registry.Route{Host: "h.example.com"}, cgroup.OverlapLimit(cgroup.TierDefault, cgroup.TierDefault)); !errors.Is(err, cgroup.ErrNoDelegation) { |
| 181 | t.Fatalf("ParentLimit with no cgroup manager: %v, want cgroup.ErrNoDelegation", err) |
| 182 | } |
| 183 | } |
| 184 | |
| 185 | // A fixture is not cgroup v2: KillGen must refuse rather than claim an |
| 186 | // empty child it never proved. |
| 187 | func TestKillGenFlatRefusesWithoutCgroupV2(t *testing.T) { |
| 188 | const host = "h.example.com" |
| 189 | b, parent := delegatedBackend(t, host) |
| 190 | b.GenContainment = true |
| 191 | if err := os.Mkdir(filepath.Join(parent, "g7"), 0o755); err != nil { |
| 192 | t.Fatal(err) |
| 193 | } |
| 194 | if err := b.KillGen(context.Background(), registry.Route{Host: host}, 7); err == nil { |
| 195 | t.Fatal("KillGen reported an unproven kill as done") |
| 196 | } |
| 197 | } |
| 198 | |
| 199 | func TestParentLimitAndKillGenNestedAskTheControlService(t *testing.T) { |
| 200 | ctl := &fakeControl{} |
| 201 | b, rt := nestedBackend(t, ctl, &fakeUnit{}) |
| 202 | b.GenContainment = true |
| 203 | b.Control = fakeGenControl{ctl} |
| 204 | if err := b.ParentLimit(context.Background(), rt, cgroup.OverlapLimit(cgroup.TierDefault, cgroup.TierDefault)); err != nil { |
| 205 | t.Fatal(err) |
| 206 | } |
| 207 | if err := b.KillGen(context.Background(), rt, 7); err != nil { |
| 208 | t.Fatal(err) |
| 209 | } |
| 210 | if want := "parent h.example.com 1073741824\nstopgen h.example.com 7 kill=true"; ctl.joined() != want { |
| 211 | t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want) |
| 212 | } |
| 213 | b.Control = ctl // a Control without the generation verbs |
| 214 | if err := b.ParentLimit(context.Background(), rt, cgroup.TierDefault); !errors.Is(err, errNoGenControl) { |
| 215 | t.Errorf("ParentLimit through a Control without generations: %v", err) |
| 216 | } |
| 217 | } |
| 218 | |
| 219 | // SplitBackend forwards KillGen to the exec half for an exec row. A |
| 220 | // unit-backed row, and an exec half that cannot kill a generation, refuse: |
| 221 | // a caller must never believe a kill it could not make. |
| 222 | func TestSplitBackendKillGen(t *testing.T) { |
| 223 | ctl := &fakeControl{} |
| 224 | b, rt := nestedBackend(t, ctl, &fakeUnit{}) |
| 225 | b.GenContainment = true |
| 226 | b.Control = fakeGenControl{ctl} |
| 227 | sb := &SplitBackend{Exec: b, Unit: &UnitBackend{}} |
| 228 | if err := sb.KillGen(context.Background(), rt, 7); err != nil { |
| 229 | t.Fatal(err) |
| 230 | } |
| 231 | if want := "stopgen h.example.com 7 kill=true"; ctl.joined() != want { |
| 232 | t.Errorf("calls =\n%s\nwant\n%s", ctl.joined(), want) |
| 233 | } |
| 234 | unitRt := rt |
| 235 | unitRt.Backing = registry.BackingUnit |
| 236 | if err := sb.KillGen(context.Background(), unitRt, 7); !errors.Is(err, errNoGenContainment) { |
| 237 | t.Errorf("KillGen of a unit-backed row: %v", err) |
| 238 | } |
| 239 | if want := "stopgen h.example.com 7 kill=true"; ctl.joined() != want { |
| 240 | t.Errorf("a unit-backed KillGen reached the control service:\n%s", ctl.joined()) |
| 241 | } |
| 242 | plain := &SplitBackend{Exec: &fakeBackend{}, Unit: &UnitBackend{}} |
| 243 | if err := plain.KillGen(context.Background(), rt, 7); !errors.Is(err, errNoGenContainment) { |
| 244 | t.Errorf("KillGen through an exec half without KillGen: %v", err) |
| 245 | } |
| 246 | } |
| 247 | |
| 248 | // A placed spawn that fails to start leaves no child cgroup behind. |
| 249 | func TestSpawnInCgroupAtRemovesTheChildOfAFailedSpawn(t *testing.T) { |
| 250 | const host = "h.example.com" |
| 251 | b, parent := delegatedBackend(t, host) |
| 252 | b.GenContainment = true |
| 253 | // A zero tier writes no limit files, so the fixture's child directory is |
| 254 | // as empty as a real cgroup's after its control files vanish. |
| 255 | b.Tier = func(registry.Route) cgroup.Limit { return cgroup.Limit{} } |
| 256 | _, _, err := b.spawnInCgroupAt(registry.Route{Host: host}, "g7", true, func() *exec.Cmd { |
| 257 | return exec.Command(filepath.Join(t.TempDir(), "no-such-binary")) |
| 258 | }) |
| 259 | if err == nil { |
| 260 | t.Fatal("a spawn of a missing binary succeeded") |
| 261 | } |
| 262 | if _, err := os.Stat(filepath.Join(parent, "g7")); !os.IsNotExist(err) { |
| 263 | t.Errorf("the failed spawn's child cgroup is still there: %v", err) |
| 264 | } |
| 265 | } |
| 266 | |
| 267 | // A placed spawn never starts outside a strict host's group: a host that is |
| 268 | // strict, or one whose group a strict owner holds, is refused. |
| 269 | func TestSpawnInCgroupAtRefusesAStrictHost(t *testing.T) { |
| 270 | const host = "h.example.com" |
| 271 | b, parent := delegatedBackend(t, host) |
| 272 | b.GenContainment = true |
| 273 | b.Strict = func(string) bool { return true } |
| 274 | build := func() *exec.Cmd { return exec.Command("/bin/true") } |
| 275 | if _, _, err := b.spawnInCgroupAt(registry.Route{Host: host}, "g7", true, build); !errors.Is(err, ErrStrictRequired) { |
| 276 | t.Fatalf("a strict host's placed spawn: %v", err) |
| 277 | } |
| 278 | b.Strict = func(string) bool { return false } |
| 279 | b.gate(host).owner = &strictOwner{} |
| 280 | if _, _, err := b.spawnInCgroupAt(registry.Route{Host: host}, "role-seed", false, build); !errors.Is(err, ErrStrictGroupInUse) { |
| 281 | t.Fatalf("a placed spawn beside a strict owner: %v", err) |
| 282 | } |
| 283 | if _, err := os.Stat(filepath.Join(parent, "g7")); !os.IsNotExist(err) { |
| 284 | t.Errorf("a refused spawn made a child cgroup: %v", err) |
| 285 | } |
| 286 | } |
| 287 | |
| 288 | // Amadan #272: a host whose parent cgroup still holds a process directly |
| 289 | // (a legacy flat start) gets no generation, in any child shape. The refusal |
| 290 | // comes back before the exec, leaves no child, and reads as nothing of the |
| 291 | // build having run, so no swap or wake counts it as "may have run" or as a |
| 292 | // failure of the build. |
| 293 | func TestSpawnInCgroupAtRefusesAPopulatedLegacyParent(t *testing.T) { |
| 294 | const host = "h.example.com" |
| 295 | b, parent := delegatedBackend(t, host) |
| 296 | b.GenContainment = true |
| 297 | if err := os.WriteFile(filepath.Join(parent, "cgroup.procs"), []byte("4242\n"), 0o644); err != nil { |
| 298 | t.Fatal(err) |
| 299 | } |
| 300 | ran := false |
| 301 | build := func() *exec.Cmd { ran = true; return exec.Command("/bin/true") } |
| 302 | for _, child := range []string{"g7", "role-seed", "role-sidecar"} { |
| 303 | cmd, _, err := b.spawnInCgroupAt(registry.Route{Host: host}, child, child == "g7", build) |
| 304 | if !errors.Is(err, cgroup.ErrLegacyParentPopulated) || cmd != nil { |
| 305 | t.Fatalf("%s under a populated parent: %v, %v", child, cmd, err) |
| 306 | } |
| 307 | if startMayHaveRun(nil, err) { |
| 308 | t.Errorf("%s: the refusal reads as a start that may have run", child) |
| 309 | } |
| 310 | if verdictOf(err) != gateNoOwnership { |
| 311 | t.Errorf("%s: the refusal is not an abandonment", child) |
| 312 | } |
| 313 | if _, serr := os.Stat(filepath.Join(parent, child)); !os.IsNotExist(serr) { |
| 314 | t.Errorf("%s: a refused spawn made a child cgroup: %v", child, serr) |
| 315 | } |
| 316 | } |
| 317 | if ran { |
| 318 | t.Error("the build was prepared for exec under a populated parent") |
| 319 | } |
| 320 | } |
| 321 | |
| 322 | // zdd-readiness Task 6: errStartedThenExited is found through every wrap |
| 323 | // spawnUnitFrom gives it (the uncertain mark, and beside errGenUnproven when |
| 324 | // the kill is unproven), and the text is the one it always was: a flag-off |
| 325 | // nested wake fails with it, and the member reads it. |
| 326 | func TestStartedThenExitedIsUncertainAndKeepsItsText(t *testing.T) { |
| 327 | unit := exitedUnobserved{unit: "carlos-instance-gen@h.example.com:7.service"} |
| 328 | proven := markStartUncertain(fmt.Errorf("%s: start unit: %w", "h.example.com", unit)) |
| 329 | unproven := markStartUncertain(fmt.Errorf("%s: start unit: %w; %w: %v", "h.example.com", unit, errGenUnproven, errors.New("stop timed out"))) |
| 330 | for _, err := range []error{proven, unproven} { |
| 331 | if !errors.Is(err, errStartedThenExited) || !startMayHaveRun(nil, err) { |
| 332 | t.Fatalf("%v: errStartedThenExited %v, may have run %v; want both", err, errors.Is(err, errStartedThenExited), startMayHaveRun(nil, err)) |
| 333 | } |
| 334 | } |
| 335 | if want := "h.example.com: start unit: carlos-instance-gen@h.example.com:7.service exited before it was observed"; proven.Error() != want { |
| 336 | t.Fatalf("text %q, want %q", proven.Error(), want) |
| 337 | } |
| 338 | if !genStartUnproven(unproven) || genStartUnproven(proven) { |
| 339 | t.Fatal("only the unproven kill carries errGenUnproven") |
| 340 | } |
| 341 | if errors.Is(markStartUncertain(errors.New("the unit failed to start")), errStartedThenExited) { |
| 342 | t.Fatal("another uncertain start reads as a crash at the start") |
| 343 | } |
| 344 | } |
| 345 | |
| 346 | // Task 8: a failed flat seed's processes are killed and proven gone before |
| 347 | // its database is removed. Here they cannot be (a temp dir is no cgroupfs, |
| 348 | // so the host cgroup's kill is unproven): the seed answers ErrSeedUnproven, |
| 349 | // its database stays, and a durable obligation is left beside it. |
| 350 | func TestAFailedSeedWhoseCleanupIsUnprovenKeepsTheDBAndLeavesAnObligation(t *testing.T) { |
| 351 | const host = "unproven-seed.example.com" |
| 352 | b, _ := delegatedBackend(t, host) |
| 353 | stageSeedScript(t, b, host, "v1", "printf half > \"$3\"\nexit 1\n") |
| 354 | rt := registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`} |
| 355 | |
| 356 | err := b.Seed(context.Background(), rt) |
| 357 | if !errors.Is(err, ErrSeedUnproven) { |
| 358 | t.Fatalf("Seed = %v, want ErrSeedUnproven: the seed's leftovers were never proven gone", err) |
| 359 | } |
| 360 | db := filepath.Join(b.DataDir, host+".db") |
| 361 | if got, err := os.ReadFile(db); err != nil || string(got) != "half" { |
| 362 | t.Fatalf("database = %q, %v; want it kept: a process may still be writing it", got, err) |
| 363 | } |
| 364 | if _, err := os.Stat(db + seedUnprovenSuffix); err != nil { |
| 365 | t.Fatalf("no obligation beside the database: %v", err) |
| 366 | } |
| 367 | } |
| 368 | |
| 369 | // Task 8, plan 3: the obligation survives to the next wake. While the proof |
| 370 | // still fails, Restore refuses and the database is never reported warm. |
| 371 | // Once it passes, the database, its WAL and SHM and the obligation are |
| 372 | // removed, the wake is fresh, and the seed runs again. |
| 373 | func TestTheNextWakeAfterAnUnprovenSeedProvesItBeforeTheDBIsWarm(t *testing.T) { |
| 374 | const host = "obligation.example.com" |
| 375 | b, _ := delegatedBackend(t, host) |
| 376 | runs := filepath.Join(t.TempDir(), "runs") |
| 377 | stageSeedScript(t, b, host, "v1", fmt.Sprintf( |
| 378 | "echo run >> %q\nprintf half > \"$3\"\n[ \"$(wc -l < %q)\" -ge 2 ] && exit 0\nexit 1\n", runs, runs)) |
| 379 | swapRestoreReplica(t, func(context.Context, string, string, string, string) (bool, error) { |
| 380 | return false, nil |
| 381 | }) |
| 382 | rt := registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`} |
| 383 | db := filepath.Join(b.DataDir, host+".db") |
| 384 | |
| 385 | if err := b.Seed(context.Background(), rt); !errors.Is(err, ErrSeedUnproven) { |
| 386 | t.Fatalf("first Seed = %v, want ErrSeedUnproven", err) |
| 387 | } |
| 388 | for _, p := range []string{db + "-wal", db + "-shm"} { |
| 389 | if err := os.WriteFile(p, []byte("x"), 0o600); err != nil { |
| 390 | t.Fatal(err) |
| 391 | } |
| 392 | } |
| 393 | |
| 394 | // The proof still fails: refused, and the database is not warm. |
| 395 | fresh, err := b.Restore(context.Background(), rt) |
| 396 | if !errors.Is(err, ErrSeedUnproven) || fresh { |
| 397 | t.Fatalf("Restore = %v, %v; want a refusal with ErrSeedUnproven", fresh, err) |
| 398 | } |
| 399 | if got, _ := os.ReadFile(db); string(got) != "half" { |
| 400 | t.Fatalf("database = %q after a refused wake, want it untouched", got) |
| 401 | } |
| 402 | if _, err := os.Stat(db + seedUnprovenSuffix); err != nil { |
| 403 | t.Fatalf("the obligation went with a refused wake: %v", err) |
| 404 | } |
| 405 | |
| 406 | // The proof passes: everything of the failed seed goes, and it is fresh. |
| 407 | var asked []string |
| 408 | b.cgroupKill = func(_ context.Context, _ registry.Route, child string) error { |
| 409 | asked = append(asked, child) |
| 410 | return nil |
| 411 | } |
| 412 | fresh, err = b.Restore(context.Background(), rt) |
| 413 | if err != nil || !fresh { |
| 414 | t.Fatalf("Restore = %v, %v; want fresh once the proof holds", fresh, err) |
| 415 | } |
| 416 | if len(asked) != 1 || asked[0] != "" { |
| 417 | t.Fatalf("proof asked of %q, want the host cgroup once", asked) |
| 418 | } |
| 419 | for _, p := range []string{db, db + "-wal", db + "-shm", db + seedUnprovenSuffix} { |
| 420 | if _, err := os.Stat(p); !os.IsNotExist(err) { |
| 421 | t.Fatalf("%s still there (%v) after the proof held", filepath.Base(p), err) |
| 422 | } |
| 423 | } |
| 424 | if err := b.Seed(context.Background(), rt); err != nil { |
| 425 | t.Fatalf("second Seed: %v", err) |
| 426 | } |
| 427 | got, err := os.ReadFile(runs) |
| 428 | if err != nil || strings.Count(string(got), "run\n") != 2 { |
| 429 | t.Fatalf("seed runs = %q, %v; want the seed to have run twice", got, err) |
| 430 | } |
| 431 | } |
| 432 | |
| 433 | // Task 8, plan 6: flag on, a successful seed's role child is killed and |
| 434 | // proven empty before the seed returns, so a helper that redirected its |
| 435 | // output (Wait returns a plain nil) does not outlive it. |
| 436 | func TestAFlagOnSeedKillsAHelperThatOutlivesIt(t *testing.T) { |
| 437 | const host = "helper-seed.example.com" |
| 438 | b, _ := delegatedBackend(t, host) |
| 439 | cgroup.FakePlacementForTest(t) // a temp dir is no cgroupfs, and a role child is never moved by PID |
| 440 | b.GenContainment = true |
| 441 | pidFile := filepath.Join(t.TempDir(), "helper.pid") |
| 442 | stageSeedScript(t, b, host, "v1", fmt.Sprintf( |
| 443 | "sleep 30 >/dev/null 2>&1 &\necho $! > %q\nprintf seeded > \"$3\"\nexit 0\n", pidFile)) |
| 444 | helper := func() int { |
| 445 | raw, err := os.ReadFile(pidFile) |
| 446 | if err != nil { |
| 447 | return 0 |
| 448 | } |
| 449 | pid, _ := strconv.Atoi(strings.TrimSpace(string(raw))) |
| 450 | return pid |
| 451 | } |
| 452 | t.Cleanup(func() { |
| 453 | if pid := helper(); pid > 0 { |
| 454 | _ = syscall.Kill(pid, syscall.SIGKILL) |
| 455 | } |
| 456 | }) |
| 457 | var asked []string |
| 458 | b.cgroupKill = func(_ context.Context, _ registry.Route, child string) error { |
| 459 | asked = append(asked, child) |
| 460 | if pid := helper(); pid > 0 { |
| 461 | _ = syscall.Kill(pid, syscall.SIGKILL) // what cgroup.kill does to the role child |
| 462 | } |
| 463 | return nil |
| 464 | } |
| 465 | |
| 466 | if err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}); err != nil { |
| 467 | t.Fatalf("Seed: %v", err) |
| 468 | } |
| 469 | if len(asked) != 1 || asked[0] != cgroup.RoleChild("seed") { |
| 470 | t.Fatalf("killed %q before the seed returned, want the seed's role child once", asked) |
| 471 | } |
| 472 | if got, err := os.ReadFile(filepath.Join(b.DataDir, host+".db")); err != nil || string(got) != "seeded" { |
| 473 | t.Fatalf("database = %q, %v; want the seed's work kept", got, err) |
| 474 | } |
| 475 | } |
| 476 | |
| 477 | // Flag on, a successful seed whose role child cannot be proven empty is |
| 478 | // unproven: the helper may still write the database, so the wake must not |
| 479 | // go on to serve it, and the next wake must prove it first. |
| 480 | func TestAFlagOnSeedWhoseHelperIsNotProvenGoneIsUnproven(t *testing.T) { |
| 481 | const host = "helper-unproven.example.com" |
| 482 | b, _ := delegatedBackend(t, host) |
| 483 | cgroup.FakePlacementForTest(t) |
| 484 | b.GenContainment = true |
| 485 | stageSeedScript(t, b, host, "v1", "printf seeded > \"$3\"\nexit 0\n") |
| 486 | b.cgroupKill = func(context.Context, registry.Route, string) error { |
| 487 | return errors.New("the role child still holds a process") |
| 488 | } |
| 489 | |
| 490 | err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}) |
| 491 | if !errors.Is(err, ErrSeedUnproven) { |
| 492 | t.Fatalf("Seed = %v, want ErrSeedUnproven", err) |
| 493 | } |
| 494 | if _, err := os.Stat(filepath.Join(b.DataDir, host+".db"+seedUnprovenSuffix)); err != nil { |
| 495 | t.Fatalf("no obligation beside the database: %v", err) |
| 496 | } |
| 497 | } |
| 498 | |
| 499 | // Task 8 fix round 1, finding 1: the obligation is written ahead. It is on |
| 500 | // disk, durably, before the seed's first instruction runs, so an agent that |
| 501 | // dies mid-seed leaves it beside the partial database, and the next wake |
| 502 | // proves the seed gone before the database can count as warm. A seed that |
| 503 | // succeeds (flag off: no proof of its own) removes it at once. |
| 504 | func TestASeedRecordsItsObligationBeforeItRuns(t *testing.T) { |
| 505 | const host = "write-ahead.example.com" |
| 506 | b, _ := delegatedBackend(t, host) |
| 507 | snap := filepath.Join(t.TempDir(), "obligation-at-run") |
| 508 | stageSeedScript(t, b, host, "v1", fmt.Sprintf( |
| 509 | "cp \"$3%s\" %q || exit 7\nprintf seeded > \"$3\"\nexit 0\n", seedUnprovenSuffix, snap)) |
| 510 | swapRestoreReplica(t, func(context.Context, string, string, string, string) (bool, error) { |
| 511 | return false, nil |
| 512 | }) |
| 513 | rt := registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`} |
| 514 | db := filepath.Join(b.DataDir, host+".db") |
| 515 | ob := db + seedUnprovenSuffix |
| 516 | |
| 517 | if err := b.Seed(context.Background(), rt); err != nil { |
| 518 | t.Fatalf("Seed = %v; want success, with the obligation there while it ran", err) |
| 519 | } |
| 520 | if _, err := os.Stat(ob); !os.IsNotExist(err) { |
| 521 | t.Fatalf("obligation still there (%v) after a successful flag-off seed", err) |
| 522 | } |
| 523 | |
| 524 | // The agent died mid-seed: what is on disk is what the seed saw while |
| 525 | // it ran, the obligation and a partial database. |
| 526 | raw, err := os.ReadFile(snap) |
| 527 | if err != nil { |
| 528 | t.Fatal(err) |
| 529 | } |
| 530 | if err := os.WriteFile(ob, raw, 0o600); err != nil { |
| 531 | t.Fatal(err) |
| 532 | } |
| 533 | if err := os.WriteFile(db, []byte("half"), 0o600); err != nil { |
| 534 | t.Fatal(err) |
| 535 | } |
| 536 | b.cgroupKill = func(context.Context, registry.Route, string) error { |
| 537 | return errors.New("the host cgroup still holds a process") |
| 538 | } |
| 539 | if fresh, err := b.Restore(context.Background(), rt); !errors.Is(err, ErrSeedUnproven) || fresh { |
| 540 | t.Fatalf("Restore = %v, %v; want a refusal while the dead agent's seed is unproven", fresh, err) |
| 541 | } |
| 542 | if got, _ := os.ReadFile(db); string(got) != "half" { |
| 543 | t.Fatalf("database = %q after a refused wake, want it untouched", got) |
| 544 | } |
| 545 | b.cgroupKill = func(context.Context, registry.Route, string) error { return nil } |
| 546 | if fresh, err := b.Restore(context.Background(), rt); err != nil || !fresh { |
| 547 | t.Fatalf("Restore = %v, %v; want fresh once the proof holds", fresh, err) |
| 548 | } |
| 549 | for _, p := range []string{db, ob} { |
| 550 | if _, err := os.Stat(p); !os.IsNotExist(err) { |
| 551 | t.Fatalf("%s still there (%v) after the proof held", filepath.Base(p), err) |
| 552 | } |
| 553 | } |
| 554 | } |
| 555 | |
| 556 | // Finding 1: an obligation that cannot be written (a full disk) refuses the |
| 557 | // seed before anything runs. Nothing ran, so this is an ordinary failure |
| 558 | // (not ErrSeedUnproven), and the wake may release its lease. |
| 559 | func TestASeedWhoseObligationCannotBeWrittenRunsNothing(t *testing.T) { |
| 560 | const host = "no-room.example.com" |
| 561 | b, _ := delegatedBackend(t, host) |
| 562 | runs := filepath.Join(t.TempDir(), "runs") |
| 563 | stageSeedScript(t, b, host, "v1", fmt.Sprintf("echo run >> %q\nprintf half > \"$3\"\nexit 0\n", runs)) |
| 564 | orig := writeSeedObligationFile |
| 565 | t.Cleanup(func() { writeSeedObligationFile = orig }) |
| 566 | writeSeedObligationFile = func(string, []byte, os.FileMode) error { return syscall.ENOSPC } |
| 567 | |
| 568 | err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}) |
| 569 | if err == nil || errors.Is(err, ErrSeedUnproven) || !errors.Is(err, syscall.ENOSPC) { |
| 570 | t.Fatalf("Seed = %v, want a refusal naming the failed write, not ErrSeedUnproven", err) |
| 571 | } |
| 572 | if _, err := os.Stat(runs); !os.IsNotExist(err) { |
| 573 | t.Fatalf("the seed ran (%v) with no obligation recorded", err) |
| 574 | } |
| 575 | if _, err := os.Stat(filepath.Join(b.DataDir, host+".db")); !os.IsNotExist(err) { |
| 576 | t.Fatalf("a database is there (%v) though no seed ran", err) |
| 577 | } |
| 578 | } |
| 579 | |
| 580 | // Findings 1 and 7: a failed seed proven gone has its database removed with |
| 581 | // every error checked, and the obligation goes only after. A database file |
| 582 | // that cannot be removed keeps the obligation (the next wake proves and |
| 583 | // removes again) and fails the wake as an ordinary failure: the seed's |
| 584 | // processes are proven gone, so the lease may go. |
| 585 | func TestAProvenFailedSeedWhoseDBCannotBeRemovedKeepsItsObligation(t *testing.T) { |
| 586 | const host = "stuck-wal.example.com" |
| 587 | b, _ := delegatedBackend(t, host) |
| 588 | stageSeedScript(t, b, host, "v1", "printf half > \"$3\"\nmkdir -p \"$3-wal/x\"\nexit 1\n") |
| 589 | b.cgroupKill = func(context.Context, registry.Route, string) error { return nil } |
| 590 | db := filepath.Join(b.DataDir, host+".db") |
| 591 | |
| 592 | err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}) |
| 593 | if err == nil || errors.Is(err, ErrSeedUnproven) { |
| 594 | t.Fatalf("Seed = %v, want the seed's own failure", err) |
| 595 | } |
| 596 | if !strings.Contains(err.Error(), "-wal") { |
| 597 | t.Fatalf("Seed = %v, want it to name the file that could not be removed", err) |
| 598 | } |
| 599 | if _, err := os.Stat(db + seedUnprovenSuffix); err != nil { |
| 600 | t.Fatalf("obligation gone (%v) though the database was not removed", err) |
| 601 | } |
| 602 | } |
| 603 | |
| 604 | // Finding 6: a database that was there before the seed is never the |
| 605 | // obligation's to remove, but it is still refused as warm while the seed |
| 606 | // beside it is unproven. Once the proof holds, the obligation goes and the |
| 607 | // database stays. |
| 608 | func TestAPreexistingDBBesideAnUnprovenSeedIsRefusedThenKept(t *testing.T) { |
| 609 | const host = "preexisting.example.com" |
| 610 | b, _ := delegatedBackend(t, host) |
| 611 | stageSeedScript(t, b, host, "v1", "exit 1\n") |
| 612 | db := filepath.Join(b.DataDir, host+".db") |
| 613 | if err := os.MkdirAll(filepath.Dir(db), 0o755); err != nil { |
| 614 | t.Fatal(err) |
| 615 | } |
| 616 | if err := os.WriteFile(db, []byte("data"), 0o600); err != nil { |
| 617 | t.Fatal(err) |
| 618 | } |
| 619 | rt := registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`} |
| 620 | |
| 621 | if err := b.Seed(context.Background(), rt); !errors.Is(err, ErrSeedUnproven) { |
| 622 | t.Fatalf("Seed = %v, want ErrSeedUnproven", err) |
| 623 | } |
| 624 | if _, err := os.Stat(db + seedUnprovenSuffix); err != nil { |
| 625 | t.Fatalf("no obligation beside a pre-existing database (%v): the next wake would serve it warm", err) |
| 626 | } |
| 627 | if fresh, err := b.Restore(context.Background(), rt); !errors.Is(err, ErrSeedUnproven) || fresh { |
| 628 | t.Fatalf("Restore = %v, %v; want a refusal while the seed is unproven", fresh, err) |
| 629 | } |
| 630 | b.cgroupKill = func(context.Context, registry.Route, string) error { return nil } |
| 631 | if fresh, err := b.Restore(context.Background(), rt); err != nil || fresh { |
| 632 | t.Fatalf("Restore = %v, %v; want the warm database once the proof holds", fresh, err) |
| 633 | } |
| 634 | if got, err := os.ReadFile(db); err != nil || string(got) != "data" { |
| 635 | t.Fatalf("database = %q, %v; want the pre-existing one kept", got, err) |
| 636 | } |
| 637 | if _, err := os.Stat(db + seedUnprovenSuffix); !os.IsNotExist(err) { |
| 638 | t.Fatalf("obligation still there (%v) after the proof held", err) |
| 639 | } |
| 640 | } |
| 641 | |
| 642 | // Finding 5, on the seed's own proof: a kernel with no cgroup.kill can kill |
| 643 | // nothing, so the leader's exit is the only proof there is, as with no |
| 644 | // delegation. Read as unproven instead, an obligation could never be |
| 645 | // discharged on such a box and the row would stay down for good. |
| 646 | func TestASeedProofWithNoCgroupKillIsTheLeaderOnlyProof(t *testing.T) { |
| 647 | const host = "old-kernel-seed.example.com" |
| 648 | b, _ := delegatedBackend(t, host) |
| 649 | b.cgroupKill = func(context.Context, registry.Route, string) error { |
| 650 | return fmt.Errorf("%w: open cgroup.kill: no such file or directory", cgroup.ErrNoKill) |
| 651 | } |
| 652 | if err := b.seedDescendantsGone(context.Background(), registry.Route{Host: host}, ""); err != nil { |
| 653 | t.Fatalf("seedDescendantsGone = %v, want the leader-only proof (nil)", err) |
| 654 | } |
| 655 | } |
| 656 | |
| 657 | // Flag on, a failed seed's role child is proven empty before the database |
| 658 | // it built is removed. |
| 659 | func TestAFlagOnFailedSeedProvesItsRoleChildBeforeRemovingTheDB(t *testing.T) { |
| 660 | const host = "failed-role-seed.example.com" |
| 661 | b, _ := delegatedBackend(t, host) |
| 662 | cgroup.FakePlacementForTest(t) |
| 663 | b.GenContainment = true |
| 664 | stageSeedScript(t, b, host, "v1", "printf half > \"$3\"\nexit 1\n") |
| 665 | db := filepath.Join(b.DataDir, host+".db") |
| 666 | var asked []string |
| 667 | b.cgroupKill = func(_ context.Context, _ registry.Route, child string) error { |
| 668 | if _, err := os.Stat(db); err != nil { |
| 669 | t.Errorf("the database was gone (%v) before the seed's processes were proven gone", err) |
| 670 | } |
| 671 | asked = append(asked, child) |
| 672 | return nil |
| 673 | } |
| 674 | |
| 675 | err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}) |
| 676 | if err == nil || errors.Is(err, ErrSeedUnproven) { |
| 677 | t.Fatalf("Seed = %v, want the seed's own failure", err) |
| 678 | } |
| 679 | if len(asked) != 1 || asked[0] != cgroup.RoleChild("seed") { |
| 680 | t.Fatalf("proved %q, want the seed's role child once", asked) |
| 681 | } |
| 682 | for _, p := range []string{db, db + seedUnprovenSuffix} { |
| 683 | if _, err := os.Stat(p); !os.IsNotExist(err) { |
| 684 | t.Fatalf("%s still there (%v) after a proven failed seed", filepath.Base(p), err) |
| 685 | } |
| 686 | } |
| 687 | } |
| 688 | |
| 689 | // Fix wave 1, item 2(a) (Astra final review 1, finding 2): a successful |
| 690 | // flag-off seed has no proof of its own (a helper it left stays in the host |
| 691 | // cgroup), so when its obligation cannot be discharged the wake fails with |
| 692 | // that helper's termination unknown. The proof is tried then: here the |
| 693 | // host cgroup still holds a process, so the seed answers ErrSeedUnproven and |
| 694 | // the lease is left to its TTL. The seed below swaps its obligation for a |
| 695 | // non-empty directory, which the discharge cannot remove. |
| 696 | func TestAFlagOffSeedWhoseDischargeFailsIsUnprovenWhileItsHelperRemains(t *testing.T) { |
| 697 | const host = "discharge-fails.example.com" |
| 698 | b, _ := delegatedBackend(t, host) |
| 699 | stageSeedScript(t, b, host, "v1", fmt.Sprintf( |
| 700 | "rm -f \"$3%[1]s\" && mkdir -p \"$3%[1]s/x\"\nprintf seeded > \"$3\"\nexit 0\n", seedUnprovenSuffix)) |
| 701 | var asked []string |
| 702 | b.cgroupKill = func(_ context.Context, _ registry.Route, child string) error { |
| 703 | asked = append(asked, child) |
| 704 | return errors.New("the host cgroup still holds a process") |
| 705 | } |
| 706 | |
| 707 | err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}) |
| 708 | if !errors.Is(err, ErrSeedUnproven) { |
| 709 | t.Fatalf("Seed = %v, want ErrSeedUnproven: the discharge failed and the seed's helper is not proven gone", err) |
| 710 | } |
| 711 | if len(asked) != 1 || asked[0] != "" { |
| 712 | t.Fatalf("proof asked of %q, want the host cgroup once", asked) |
| 713 | } |
| 714 | } |
| 715 | |
| 716 | // Fix wave 1, item 2(a), the other side: the discharge fails but the proof, |
| 717 | // tried then, holds. Nothing of the seed is left, so the failure is an |
| 718 | // ordinary one and the lease may go; the obligation stays for the next |
| 719 | // wake. |
| 720 | func TestAFlagOffSeedWhoseDischargeFailsAfterAProofIsAnOrdinaryFailure(t *testing.T) { |
| 721 | const host = "discharge-fails-proven.example.com" |
| 722 | b, _ := delegatedBackend(t, host) |
| 723 | stageSeedScript(t, b, host, "v1", fmt.Sprintf( |
| 724 | "rm -f \"$3%[1]s\" && mkdir -p \"$3%[1]s/x\"\nprintf seeded > \"$3\"\nexit 0\n", seedUnprovenSuffix)) |
| 725 | b.cgroupKill = func(context.Context, registry.Route, string) error { return nil } |
| 726 | |
| 727 | err := b.Seed(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}) |
| 728 | if err == nil || errors.Is(err, ErrSeedUnproven) { |
| 729 | t.Fatalf("Seed = %v, want an ordinary failure: the proof held", err) |
| 730 | } |
| 731 | if _, err := os.Stat(filepath.Join(b.DataDir, host+".db"+seedUnprovenSuffix)); err != nil { |
| 732 | t.Fatalf("the obligation is gone (%v); it must stay for the next wake", err) |
| 733 | } |
| 734 | } |
| 735 | |
| 736 | // Fix wave 1, item 2(b): an obligation that is there but cannot be read |
| 737 | // says nothing of whether its seed's processes are gone, so Restore refuses |
| 738 | // with ErrSeedUnproven (the lease is left to its TTL), never an ordinary |
| 739 | // error that releases it. The host cgroup here still holds a process. |
| 740 | func TestAnUnreadableObligationIsUnproven(t *testing.T) { |
| 741 | if os.Geteuid() == 0 { |
| 742 | t.Skip("root reads a mode-000 file") |
| 743 | } |
| 744 | const host = "unreadable-obligation.example.com" |
| 745 | b, _ := delegatedBackend(t, host) |
| 746 | b.cgroupKill = func(context.Context, registry.Route, string) error { |
| 747 | return errors.New("the host cgroup still holds a process") |
| 748 | } |
| 749 | db := filepath.Join(b.DataDir, host+".db") |
| 750 | if err := os.MkdirAll(filepath.Dir(db), 0o755); err != nil { |
| 751 | t.Fatal(err) |
| 752 | } |
| 753 | if err := os.WriteFile(db, []byte("half"), 0o600); err != nil { |
| 754 | t.Fatal(err) |
| 755 | } |
| 756 | ob := db + seedUnprovenSuffix |
| 757 | if err := os.WriteFile(ob, []byte(seedObligationHeader+"\n"+seedObligationOwned+"\n"+seedObligationFlat+"\n"), 0o000); err != nil { |
| 758 | t.Fatal(err) |
| 759 | } |
| 760 | t.Cleanup(func() { _ = os.Chmod(ob, 0o600) }) |
| 761 | |
| 762 | fresh, err := b.Restore(context.Background(), registry.Route{Host: host, Kind: "instance", Seed: `["seed"]`}) |
| 763 | if !errors.Is(err, ErrSeedUnproven) || fresh { |
| 764 | t.Fatalf("Restore = %v, %v; want a refusal with ErrSeedUnproven", fresh, err) |
| 765 | } |
| 766 | if got, _ := os.ReadFile(db); string(got) != "half" { |
| 767 | t.Fatalf("database = %q after a refused wake, want it untouched", got) |
| 768 | } |
| 769 | } |
| 770 | |
amadan