| 1 | package activator |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "log" |
| 8 | "os" |
| 9 | "os/exec" |
| 10 | "path/filepath" |
| 11 | "slices" |
| 12 | "strings" |
| 13 | "sync" |
| 14 | "sync/atomic" |
| 15 | "testing" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/cgroup" |
| 19 | "github.com/carlosframework/platform/internal/instancectl" |
| 20 | "github.com/carlosframework/platform/internal/registry" |
| 21 | ) |
| 22 | |
| 23 | // plantMember puts a generation into host's operation set the way Swap does |
| 24 | // (Task 4). A successor waits for the barrier's cancel and is then stopped by |
| 25 | // its owner. A retiring generation is stopped by its owner at once. |
| 26 | func plantMember(t *testing.T, a *Service, host string, role genRole, p *fakeProc, n uint64) *genMember { |
| 27 | t.Helper() |
| 28 | ctx, cancel := context.WithCancel(context.Background()) |
| 29 | t.Cleanup(cancel) |
| 30 | m := &genMember{gen: Gen{N: n, Build: "v2"}, role: role, proc: p, done: make(chan struct{})} |
| 31 | if role != roleRetiring { |
| 32 | m.cancel = cancel |
| 33 | } |
| 34 | a.mu.Lock() |
| 35 | e := a.entries[host] |
| 36 | if e == nil { |
| 37 | a.mu.Unlock() |
| 38 | t.Fatalf("no entry for %s", host) |
| 39 | } |
| 40 | a.addMemberLocked(e, m) |
| 41 | a.mu.Unlock() |
| 42 | go func() { |
| 43 | if role != roleRetiring { |
| 44 | <-ctx.Done() |
| 45 | } |
| 46 | a.dropMember(e, m) |
| 47 | }() |
| 48 | return m |
| 49 | } |
| 50 | |
| 51 | type barrierEnv struct { |
| 52 | a *Service |
| 53 | b *fakeBackend |
| 54 | clk *fakeClock |
| 55 | host string |
| 56 | } |
| 57 | |
| 58 | func awakeForBarrier(t *testing.T, swap bool) barrierEnv { |
| 59 | t.Helper() |
| 60 | a, b, clk, _ := testActivator(t, time.Hour) |
| 61 | a.gens = swap |
| 62 | host := "barrier.example.com" |
| 63 | arrangeAwake(t, a, testRoute(host)) |
| 64 | return barrierEnv{a, b, clk, host} |
| 65 | } |
| 66 | |
| 67 | // stopSignal is an EventSink that closes began when an entry's stop emits |
| 68 | // "instance stopping": the stop goroutine has started, and from there it |
| 69 | // reaches the barrier without blocking on anything else. |
| 70 | type stopSignal struct { |
| 71 | once sync.Once |
| 72 | began chan struct{} |
| 73 | } |
| 74 | |
| 75 | func (s *stopSignal) App(string, string, string, string) {} |
| 76 | func (s *stopSignal) Event(_, _, _, _, msg string) { |
| 77 | if msg == "instance stopping" { |
| 78 | s.once.Do(func() { close(s.began) }) |
| 79 | } |
| 80 | } |
| 81 | |
| 82 | // Every terminal stop cancels and joins the whole operation set before the |
| 83 | // final checkpoint and the lease release (spec §7.4). The member's stop is |
| 84 | // held open by a gate; until it closes, neither the instance's own stop |
| 85 | // nor the lease release may happen. |
| 86 | func TestSwapWitnessBarrierJoinsBeforeTheLease(t *testing.T) { |
| 87 | t.Parallel() |
| 88 | stops := []struct { |
| 89 | name string |
| 90 | stop func(x barrierEnv) |
| 91 | rewake bool |
| 92 | }{ |
| 93 | {"hibernate", func(x barrierEnv) { x.clk.advance(2 * time.Hour); x.a.sweepOnce() }, true}, |
| 94 | {"lease lost", func(x barrierEnv) { x.b.lastRep().setErr(errors.New("renewal failed")); x.a.sweepOnce() }, true}, |
| 95 | {"restart", func(x barrierEnv) { x.a.Bounce(x.host) }, true}, |
| 96 | {"maintenance fence", func(x barrierEnv) { |
| 97 | go func() { |
| 98 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) |
| 99 | defer cancel() |
| 100 | x.a.FenceAndStop(ctx, x.host, 1) //nolint:errcheck |
| 101 | }() |
| 102 | }, false}, |
| 103 | {"move or park retire", func(x barrierEnv) { x.a.RetireExcept(map[string]bool{}) }, true}, |
| 104 | {"shutdown", func(x barrierEnv) { go x.a.Shutdown(context.Background()) }, false}, |
| 105 | } |
| 106 | for _, tc := range stops { |
| 107 | for _, role := range []genRole{roleSuccessor, roleRetiring} { |
| 108 | t.Run(tc.name+"/"+role.String(), func(t *testing.T) { |
| 109 | x := awakeForBarrier(t, true) |
| 110 | sig := &stopSignal{began: make(chan struct{})} |
| 111 | x.a.Logs = sig |
| 112 | joining := make(chan struct{}) |
| 113 | var joinOnce sync.Once |
| 114 | x.a.mu.Lock() |
| 115 | x.a.joinWaiting = func(_, what string) { |
| 116 | if what == "operation set" { |
| 117 | joinOnce.Do(func() { close(joining) }) |
| 118 | } |
| 119 | } |
| 120 | x.a.mu.Unlock() |
| 121 | gate := make(chan struct{}) |
| 122 | member := &fakeProc{label: "gen:9", rec: x.b, stopGate: gate} |
| 123 | plantMember(t, x.a, x.host, role, member, 9) |
| 124 | incumbent, rep := x.b.lastProc(), x.b.lastRep() |
| 125 | tc.stop(x) |
| 126 | select { |
| 127 | case <-sig.began: |
| 128 | case <-time.After(5 * time.Second): |
| 129 | t.Fatal("the stop never began") |
| 130 | } |
| 131 | // The stop has begun. Wait for it to reach the barrier (the |
| 132 | // join seam's positive signal), or, if something skipped the |
| 133 | // barrier, for the incumbent's stop or the lease release. |
| 134 | waitFor(t, "the stop at the barrier", func() bool { |
| 135 | select { |
| 136 | case <-joining: |
| 137 | return true |
| 138 | default: |
| 139 | } |
| 140 | return rep.isClosed() || incumbent.stopEntered() > 0 |
| 141 | }) |
| 142 | if rep.isClosed() || incumbent.stopEntered() > 0 { |
| 143 | t.Fatal("the stop went past the barrier while a generation was still draining") |
| 144 | } |
| 145 | close(gate) |
| 146 | waitFor(t, "the lease release", rep.isClosed) |
| 147 | ev := x.b.events() |
| 148 | if i, j := slices.Index(ev, "stop:gen:9"), slices.Index(ev, "stop:instance"); i < 0 || j < 0 || i > j { |
| 149 | t.Fatalf("order %v: the member must stop before the instance", ev) |
| 150 | } |
| 151 | if tc.rewake { |
| 152 | // #231: Owns stays true while an entry drains, so the stop is |
| 153 | // proven by a fresh spawn instead. |
| 154 | arrangeAwake(t, x.a, testRoute(x.host)) |
| 155 | if got := x.b.startCount(); got != 2 { |
| 156 | t.Fatalf("Start calls = %d after the stop, want 2", got) |
| 157 | } |
| 158 | } |
| 159 | }) |
| 160 | } |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | // A member whose termination cannot be proven (ErrStillRunning) keeps the |
| 165 | // lease, exactly as an unproven main process does (spec §7.4). The reaper |
| 166 | // finishes the member once it exits, and then the stop completes. |
| 167 | func TestBarrierKeepsTheLeaseWhileAGenerationIsUnproven(t *testing.T) { |
| 168 | x := awakeForBarrier(t, true) |
| 169 | old := opJoinBound |
| 170 | opJoinBound = 100 * time.Millisecond |
| 171 | t.Cleanup(func() { opJoinBound = old }) |
| 172 | member := &fakeProc{label: "gen:9", rec: x.b, stuck: true} |
| 173 | plantMember(t, x.a, x.host, roleRetiring, member, 9) |
| 174 | waitFor(t, "the member marked unproven", func() bool { |
| 175 | x.a.mu.Lock() |
| 176 | defer x.a.mu.Unlock() |
| 177 | m := x.a.entries[x.host].ops.members[9] |
| 178 | return m != nil && m.unproven |
| 179 | }) |
| 180 | rep := x.b.lastRep() |
| 181 | x.a.Bounce(x.host) |
| 182 | waitFor(t, "the stop to pend", func() bool { |
| 183 | x.a.mu.Lock() |
| 184 | defer x.a.mu.Unlock() |
| 185 | e := x.a.entries[x.host] |
| 186 | return e != nil && e.pendingStop |
| 187 | }) |
| 188 | if rep.isClosed() { |
| 189 | t.Fatal("the lease was released over an unproven generation") |
| 190 | } |
| 191 | member.exit() |
| 192 | x.a.sweepOnce() // reaps the member |
| 193 | waitFor(t, "the member reaped", func() bool { |
| 194 | x.a.mu.Lock() |
| 195 | defer x.a.mu.Unlock() |
| 196 | e := x.a.entries[x.host] |
| 197 | return e == nil || len(e.ops.members) == 0 |
| 198 | }) |
| 199 | x.a.sweepOnce() // finishes the stop |
| 200 | waitFor(t, "the lease release", rep.isClosed) |
| 201 | arrangeAwake(t, x.a, testRoute(x.host)) |
| 202 | if got := x.b.startCount(); got != 2 { |
| 203 | t.Fatalf("Start calls = %d, want 2", got) |
| 204 | } |
| 205 | } |
| 206 | |
| 207 | // killGenBackend is fakeBackend plus plan 2's KillGen. err, while set, is |
| 208 | // what every kill answers: a generation child that still holds a process. |
| 209 | type killGenBackend struct { |
| 210 | *fakeBackend |
| 211 | kmu sync.Mutex |
| 212 | err error |
| 213 | kills []uint64 |
| 214 | } |
| 215 | |
| 216 | func (k *killGenBackend) KillGen(_ context.Context, _ registry.Route, n uint64) error { |
| 217 | k.kmu.Lock() |
| 218 | defer k.kmu.Unlock() |
| 219 | k.kills = append(k.kills, n) |
| 220 | return k.err |
| 221 | } |
| 222 | |
| 223 | func (k *killGenBackend) setErr(err error) { |
| 224 | k.kmu.Lock() |
| 225 | k.err = err |
| 226 | k.kmu.Unlock() |
| 227 | } |
| 228 | |
| 229 | func (k *killGenBackend) killed() []uint64 { |
| 230 | k.kmu.Lock() |
| 231 | defer k.kmu.Unlock() |
| 232 | return slices.Clone(k.kills) |
| 233 | } |
| 234 | |
| 235 | // A generation's leader can exit while a process it forked lives on in its |
| 236 | // child cgroup. The leader's exit is not termination (spec §7.3, §7.4). |
| 237 | // After the graceful drain the member's own child is killed and proven |
| 238 | // empty through plan 2's KillGen. While that proof fails, the member stays |
| 239 | // in the set and the lease stays held, exactly as for ErrStillRunning. The |
| 240 | // reaper retries the kill and finishes the stop once the child is empty. |
| 241 | // |
| 242 | // Without the KillGen step, stopMember accepts Exited() alone: the member |
| 243 | // leaves the set at once, "marked unproven" never happens, and the Bounce |
| 244 | // releases the lease over the surviving descendant. |
| 245 | func TestSwapWitnessMemberStopProvesTheGenerationChildEmpty(t *testing.T) { |
| 246 | x := awakeForBarrier(t, true) |
| 247 | old := opJoinBound |
| 248 | opJoinBound = 100 * time.Millisecond |
| 249 | t.Cleanup(func() { opJoinBound = old }) |
| 250 | kb := &killGenBackend{fakeBackend: x.b, err: errors.New("g9 still holds a process")} |
| 251 | x.a.backend = kb |
| 252 | member := &fakeProc{label: "gen:9", rec: x.b} // its leader exits on SIGTERM |
| 253 | plantMember(t, x.a, x.host, roleRetiring, member, 9) |
| 254 | waitFor(t, "the member marked unproven", func() bool { |
| 255 | x.a.mu.Lock() |
| 256 | defer x.a.mu.Unlock() |
| 257 | m := x.a.entries[x.host].ops.members[9] |
| 258 | return m != nil && m.unproven |
| 259 | }) |
| 260 | if !member.isStopped() || !member.Exited() { |
| 261 | t.Fatal("the graceful stop never ran, or the leader did not exit") |
| 262 | } |
| 263 | if got := kb.killed(); !slices.Equal(got, []uint64{9}) { |
| 264 | t.Fatalf("KillGen calls = %v, want [9] (the member's own generation)", got) |
| 265 | } |
| 266 | rep := x.b.lastRep() |
| 267 | x.a.Bounce(x.host) |
| 268 | waitFor(t, "the stop to pend", func() bool { |
| 269 | x.a.mu.Lock() |
| 270 | defer x.a.mu.Unlock() |
| 271 | e := x.a.entries[x.host] |
| 272 | return e != nil && e.pendingStop |
| 273 | }) |
| 274 | if rep.isClosed() { |
| 275 | t.Fatal("the lease was released while generation 9's child still held a process") |
| 276 | } |
| 277 | x.a.sweepOnce() // the reaper retries the kill, which still fails |
| 278 | waitFor(t, "the reaper's retry to finish", func() bool { |
| 279 | x.a.mu.Lock() |
| 280 | defer x.a.mu.Unlock() |
| 281 | m := x.a.entries[x.host].ops.members[9] |
| 282 | return len(kb.killed()) >= 2 && m != nil && !m.reaping |
| 283 | }) |
| 284 | if rep.isClosed() { |
| 285 | t.Fatal("the lease was released on a failed retry") |
| 286 | } |
| 287 | kb.setErr(nil) |
| 288 | x.a.sweepOnce() // the kill proves the child empty now |
| 289 | waitFor(t, "the member reaped", func() bool { |
| 290 | x.a.mu.Lock() |
| 291 | defer x.a.mu.Unlock() |
| 292 | e := x.a.entries[x.host] |
| 293 | return e == nil || len(e.ops.members) == 0 |
| 294 | }) |
| 295 | x.a.sweepOnce() // finishes the stop |
| 296 | waitFor(t, "the lease release", rep.isClosed) |
| 297 | } |
| 298 | |
| 299 | // A backend with no generation children (containment off answers |
| 300 | // errNoGenContainment) has nothing more to prove than the leader's exit: |
| 301 | // the generation shares the host cgroup, whose emptiness the terminal |
| 302 | // stop's own evidence proves. |
| 303 | func TestMemberStopWithoutContainmentAcceptsTheLeaderExit(t *testing.T) { |
| 304 | x := awakeForBarrier(t, true) |
| 305 | kb := &killGenBackend{fakeBackend: x.b, err: fmt.Errorf("wrapped: %w", errNoGenContainment)} |
| 306 | x.a.backend = kb |
| 307 | m := plantMember(t, x.a, x.host, roleRetiring, &fakeProc{label: "gen:9", rec: x.b}, 9) |
| 308 | select { |
| 309 | case <-m.done: |
| 310 | case <-time.After(5 * time.Second): |
| 311 | t.Fatal("the member never left the set") |
| 312 | } |
| 313 | } |
| 314 | |
| 315 | // A member that never got a process (its spawn failed with its kill |
| 316 | // unproven, as startUnitAt answers) is not proven by the missing handle: |
| 317 | // its generation's child is, through KillGen (P2-13 (f)). |
| 318 | func TestMemberStopWithoutAProcessProvesItsChild(t *testing.T) { |
| 319 | x := awakeForBarrier(t, true) |
| 320 | kb := &killGenBackend{fakeBackend: x.b, err: errors.New("g12 still holds a process")} |
| 321 | x.a.backend = kb |
| 322 | m := &genMember{gen: Gen{N: 12, Build: "v2"}, role: roleSpawning, done: make(chan struct{})} |
| 323 | x.a.mu.Lock() |
| 324 | e := x.a.entries[x.host] |
| 325 | x.a.addMemberLocked(e, m) |
| 326 | x.a.mu.Unlock() |
| 327 | if x.a.dropMember(e, m) { |
| 328 | t.Fatal("a member with no process and an unproven child was taken as terminated") |
| 329 | } |
| 330 | if got := kb.killed(); !slices.Equal(got, []uint64{12}) { |
| 331 | t.Fatalf("KillGen calls = %v, want [12]", got) |
| 332 | } |
| 333 | kb.setErr(nil) |
| 334 | x.a.sweepOnce() |
| 335 | select { |
| 336 | case <-m.done: |
| 337 | case <-time.After(5 * time.Second): |
| 338 | t.Fatal("the member never left once its child was empty") |
| 339 | } |
| 340 | } |
| 341 | |
| 342 | // stubbornProc is a main process whose Stop fails with an error other than |
| 343 | // ErrStillRunning (a claims-on KillAndEmpty error) and whose leader lives |
| 344 | // on until exit. stops counts Stop calls. |
| 345 | type stubbornProc struct { |
| 346 | exited atomic.Bool |
| 347 | stops atomic.Int32 |
| 348 | } |
| 349 | |
| 350 | func (p *stubbornProc) Stop(context.Context) error { |
| 351 | p.stops.Add(1) |
| 352 | return errors.New("cgroup.kill: the child is not empty") |
| 353 | } |
| 354 | func (p *stubbornProc) Exited() bool { return p.exited.Load() } |
| 355 | func (p *stubbornProc) Crashed() bool { return false } |
| 356 | |
| 357 | // THE retain-until-proven rule (spec §7.4): a stop whose main process is |
| 358 | // still alive after a failed kill keeps its lease. Only ErrStillRunning |
| 359 | // kept it before; any other Stop error, with the leader alive, skipped the |
| 360 | // proof (it ran only for an exited leader) and released the lease. |
| 361 | func TestAStopKeepsTheLeaseWhileItsLeaderLivesOn(t *testing.T) { |
| 362 | x := awakeForBarrier(t, true) |
| 363 | p := &stubbornProc{} |
| 364 | x.a.mu.Lock() |
| 365 | x.a.entries[x.host].proc = p |
| 366 | x.a.mu.Unlock() |
| 367 | rep := x.b.lastRep() |
| 368 | x.a.Bounce(x.host) |
| 369 | waitFor(t, "the stop to pend", func() bool { |
| 370 | x.a.mu.Lock() |
| 371 | defer x.a.mu.Unlock() |
| 372 | e := x.a.entries[x.host] |
| 373 | return e != nil && e.pendingStop |
| 374 | }) |
| 375 | // The leader is alive and the retry cadence has not come round (the |
| 376 | // clock is fake), so this sweep starts nothing: the stop stays pending. |
| 377 | x.a.sweepOnce() |
| 378 | x.a.mu.Lock() |
| 379 | e := x.a.entries[x.host] |
| 380 | idle := e != nil && e.pendingStop && !e.genReaping && !e.stopRetrying |
| 381 | x.a.mu.Unlock() |
| 382 | if !idle || rep.isClosed() { |
| 383 | t.Fatal("the lease was released while the leader still ran after a failed kill") |
| 384 | } |
| 385 | p.exited.Store(true) |
| 386 | x.a.sweepOnce() |
| 387 | waitFor(t, "the lease release", rep.isClosed) |
| 388 | } |
| 389 | |
| 390 | // A stop keeps its lease while a stray sidecar of the host is tracked |
| 391 | // (spec §7.4): its process may still run against the database. The lease |
| 392 | // went with the stop before. |
| 393 | func TestAStopKeepsTheLeaseWhileAStraySidecarRuns(t *testing.T) { |
| 394 | t.Parallel() |
| 395 | x := awakeForBarrier(t, true) |
| 396 | stray := &fakeProc{label: "sidecar", rec: x.b, stuck: true} |
| 397 | x.a.mu.Lock() |
| 398 | x.a.straySidecars = map[string][]*straySidecar{x.host: {{proc: stray, rt: testRoute(x.host)}}} |
| 399 | x.a.mu.Unlock() |
| 400 | rep := x.b.lastRep() |
| 401 | x.a.Bounce(x.host) |
| 402 | waitFor(t, "the stop to pend", func() bool { |
| 403 | x.a.mu.Lock() |
| 404 | defer x.a.mu.Unlock() |
| 405 | e := x.a.entries[x.host] |
| 406 | return e != nil && e.pendingStop |
| 407 | }) |
| 408 | if rep.isClosed() { |
| 409 | t.Fatal("the lease was released while a stray sidecar ran") |
| 410 | } |
| 411 | x.a.mu.Lock() |
| 412 | delete(x.a.straySidecars, x.host) // proven gone (Task 7's reaper does this) |
| 413 | x.a.mu.Unlock() |
| 414 | x.a.sweepOnce() |
| 415 | waitFor(t, "the lease release", rep.isClosed) |
| 416 | } |
| 417 | |
| 418 | // failingContainment is a claims-on child whose KillAndEmpty fails until |
| 419 | // heal is called. |
| 420 | type failingContainment struct { |
| 421 | mu sync.Mutex |
| 422 | bad bool |
| 423 | } |
| 424 | |
| 425 | func (c *failingContainment) KillAndEmpty(context.Context) error { |
| 426 | c.mu.Lock() |
| 427 | defer c.mu.Unlock() |
| 428 | if c.bad { |
| 429 | return errors.New("cgroup.kill: the child is not empty") |
| 430 | } |
| 431 | return nil |
| 432 | } |
| 433 | func (c *failingContainment) Close() error { return nil } |
| 434 | func (c *failingContainment) heal() { |
| 435 | c.mu.Lock() |
| 436 | c.bad = false |
| 437 | c.mu.Unlock() |
| 438 | } |
| 439 | |
| 440 | // A claims-on generation is terminated only when its own Stop completed |
| 441 | // (spec §7.4): KillAndEmpty on its child included. Here the leader has |
| 442 | // exited but KillAndEmpty fails, so the member stays in the set, unproven, |
| 443 | // until the child is empty. Treating any claimProcess as proven accepted |
| 444 | // the leader's exit alone. |
| 445 | func TestAClaimsOnMemberNeedsItsChildEmptied(t *testing.T) { |
| 446 | x := awakeForBarrier(t, true) |
| 447 | cmd := exec.Command("true") |
| 448 | if err := cmd.Run(); err != nil { |
| 449 | t.Skipf("no true(1): %v", err) |
| 450 | } |
| 451 | done := make(chan struct{}) |
| 452 | close(done) // the leader has exited |
| 453 | cont := &failingContainment{bad: true} |
| 454 | p := &claimProcess{cmd: cmd, prepared: cont, done: done} |
| 455 | m := &genMember{gen: Gen{N: 11, Build: "v2"}, role: roleRetiring, proc: p, done: make(chan struct{})} |
| 456 | x.a.mu.Lock() |
| 457 | e := x.a.entries[x.host] |
| 458 | x.a.addMemberLocked(e, m) |
| 459 | x.a.mu.Unlock() |
| 460 | if x.a.dropMember(e, m) { |
| 461 | t.Fatal("a claims-on generation whose KillAndEmpty failed was taken as terminated") |
| 462 | } |
| 463 | cont.heal() |
| 464 | x.a.sweepOnce() |
| 465 | select { |
| 466 | case <-m.done: |
| 467 | case <-time.After(5 * time.Second): |
| 468 | t.Fatal("the member never left once its child was empty") |
| 469 | } |
| 470 | } |
| 471 | |
| 472 | // The current generation is held to the same proof (spec §7.4): a terminal |
| 473 | // stop whose main process has exited keeps the lease while that |
| 474 | // generation's child is not proven empty, and finishes once it is. A stop |
| 475 | // that accepts the leader's exit alone releases the lease over a surviving |
| 476 | // descendant. This is the witness for mutation M8. |
| 477 | func TestSwapWitnessStopProvesTheCurrentGenerationEmpty(t *testing.T) { |
| 478 | t.Parallel() |
| 479 | x := awakeForBarrier(t, true) |
| 480 | kb := &killGenBackend{fakeBackend: x.b, err: errors.New("the child still holds a process")} |
| 481 | x.a.backend = kb |
| 482 | x.a.mu.Lock() |
| 483 | x.a.entries[x.host].gen.N = 1 << 41 // the woken generation, as plan 1's wake records it |
| 484 | x.a.mu.Unlock() |
| 485 | rep := x.b.lastRep() |
| 486 | x.a.Bounce(x.host) |
| 487 | waitFor(t, "the stop to pend", func() bool { |
| 488 | x.a.mu.Lock() |
| 489 | defer x.a.mu.Unlock() |
| 490 | e := x.a.entries[x.host] |
| 491 | return e != nil && e.pendingStop |
| 492 | }) |
| 493 | if !x.b.lastProc().isStopped() { |
| 494 | t.Fatal("the graceful stop never ran") |
| 495 | } |
| 496 | kills := len(kb.killed()) |
| 497 | x.a.sweepOnce() // retries the proof, which still fails |
| 498 | waitFor(t, "the retried proof to finish", func() bool { |
| 499 | x.a.mu.Lock() |
| 500 | defer x.a.mu.Unlock() |
| 501 | e := x.a.entries[x.host] |
| 502 | return len(kb.killed()) > kills && e != nil && !e.genReaping |
| 503 | }) |
| 504 | if rep.isClosed() { |
| 505 | t.Fatal("the lease was released while the current generation's child still held a process") |
| 506 | } |
| 507 | kb.setErr(nil) |
| 508 | x.a.sweepOnce() |
| 509 | waitFor(t, "the lease release", rep.isClosed) |
| 510 | if k := kb.killed(); len(k) == 0 || k[len(k)-1] != 1<<41 { |
| 511 | t.Fatalf("KillGen calls = %v, want the current generation's", k) |
| 512 | } |
| 513 | } |
| 514 | |
| 515 | // With swaps on, Shutdown covers a waking entry (spec §7.4): it cancels the |
| 516 | // wake and waits for it. Its child is stopped and its lease given back |
| 517 | // before Shutdown returns. |
| 518 | func TestSwapWitnessShutdownJoinsAWakingEntry(t *testing.T) { |
| 519 | t.Parallel() |
| 520 | a, b, _, _ := testActivator(t, time.Hour) |
| 521 | a.gens = true |
| 522 | gate := make(chan struct{}) |
| 523 | b.startGate = gate |
| 524 | host := "waking.example.com" |
| 525 | go func() { |
| 526 | if release, err := a.beginRequest(context.Background(), testRoute(host)); err == nil { |
| 527 | release() |
| 528 | } |
| 529 | }() |
| 530 | waitFor(t, "the wake to reach Start", func() bool { return b.startCount() == 1 }) |
| 531 | done := make(chan struct{}) |
| 532 | go func() { a.Shutdown(context.Background()); close(done) }() |
| 533 | select { |
| 534 | case <-done: |
| 535 | t.Fatal("Shutdown returned while a wake was still spawning") |
| 536 | case <-time.After(100 * time.Millisecond): |
| 537 | } |
| 538 | close(gate) |
| 539 | select { |
| 540 | case <-done: |
| 541 | case <-time.After(5 * time.Second): |
| 542 | t.Fatal("Shutdown never returned") |
| 543 | } |
| 544 | if !b.lastProc().isStopped() { |
| 545 | t.Fatal("the woken child outlived Shutdown") |
| 546 | } |
| 547 | if got := b.releaseCount(); got != 1 { |
| 548 | t.Fatalf("lease releases = %d, want 1", got) |
| 549 | } |
| 550 | // Shutdown cancelled the wake's context. The release must not run under |
| 551 | // it: a real release is an S3 DeleteObject that would fail at once and |
| 552 | // leave the lease held for its whole TTL. |
| 553 | for _, err := range b.releaseCtxErrors() { |
| 554 | if err != nil { |
| 555 | t.Fatalf("the lease was released under a dead context (%v)", err) |
| 556 | } |
| 557 | } |
| 558 | } |
| 559 | |
| 560 | // Every doWake bail-out gives the lease back under a live context, even |
| 561 | // after the wake's own context is gone (a fence, a Shutdown). Here a fence |
| 562 | // lands while Start is held, so the wake's context is cancelled before it |
| 563 | // releases. Flag off: the fix is not gated on generations. |
| 564 | func TestAFencedWakeReleasesUnderALiveContext(t *testing.T) { |
| 565 | t.Parallel() |
| 566 | a, b, _, _ := testActivator(t, time.Hour) |
| 567 | gate := make(chan struct{}) |
| 568 | b.startGate = gate |
| 569 | host := "fenced-wake.example.com" |
| 570 | failed := make(chan error, 1) |
| 571 | go func() { |
| 572 | release, err := a.beginRequest(context.Background(), testRoute(host)) |
| 573 | if err == nil { |
| 574 | release() |
| 575 | } |
| 576 | failed <- err |
| 577 | }() |
| 578 | waitFor(t, "the wake to reach Start", func() bool { return b.startCount() == 1 }) |
| 579 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) |
| 580 | defer cancel() |
| 581 | fenceDone := make(chan struct{}) |
| 582 | go func() { |
| 583 | a.FenceAndStop(ctx, host, 1) //nolint:errcheck |
| 584 | close(fenceDone) |
| 585 | }() |
| 586 | waitFor(t, "the fence", func() bool { return a.Fenced(host) }) |
| 587 | close(gate) |
| 588 | if err := <-failed; err == nil { |
| 589 | t.Fatal("a fenced wake succeeded") |
| 590 | } |
| 591 | <-fenceDone |
| 592 | if got := b.releaseCount(); got != 1 { |
| 593 | t.Fatalf("lease releases = %d, want 1", got) |
| 594 | } |
| 595 | for _, err := range b.releaseCtxErrors() { |
| 596 | if err != nil { |
| 597 | t.Fatalf("the lease was released under a dead context (%v)", err) |
| 598 | } |
| 599 | } |
| 600 | } |
| 601 | |
| 602 | // Flag off: Shutdown joins a waking entry exactly as it does with swaps on |
| 603 | // (#292). It used to leave one to resolve on its own, and the agent then |
| 604 | // exited with the wake still holding its lease. Here the start ignores its |
| 605 | // cancelled context, so Shutdown must wait it out, stop what it woke and |
| 606 | // give the lease back under a live context. |
| 607 | func TestShutdownJoinsAWakingEntryWithSwapOff(t *testing.T) { |
| 608 | t.Parallel() |
| 609 | a, b, _, _ := testActivator(t, time.Hour) |
| 610 | gate := make(chan struct{}) |
| 611 | b.startGate = gate |
| 612 | host := "waking.example.com" |
| 613 | go func() { |
| 614 | if release, err := a.beginRequest(context.Background(), testRoute(host)); err == nil { |
| 615 | release() |
| 616 | } |
| 617 | }() |
| 618 | waitFor(t, "the wake to reach Start", func() bool { return b.startCount() == 1 }) |
| 619 | done := make(chan struct{}) |
| 620 | go func() { a.Shutdown(context.Background()); close(done) }() |
| 621 | select { |
| 622 | case <-done: |
| 623 | t.Fatal("Shutdown returned while a wake was still spawning, with swaps off") |
| 624 | case <-time.After(100 * time.Millisecond): |
| 625 | } |
| 626 | close(gate) |
| 627 | select { |
| 628 | case <-done: |
| 629 | case <-time.After(5 * time.Second): |
| 630 | t.Fatal("Shutdown never returned") |
| 631 | } |
| 632 | if !b.lastProc().isStopped() { |
| 633 | t.Fatal("the woken child outlived Shutdown") |
| 634 | } |
| 635 | if got := b.releaseCount(); got != 1 { |
| 636 | t.Fatalf("lease releases = %d, want 1", got) |
| 637 | } |
| 638 | for _, err := range b.releaseCtxErrors() { |
| 639 | if err != nil { |
| 640 | t.Fatalf("the lease was released under a dead context (%v)", err) |
| 641 | } |
| 642 | } |
| 643 | if n := a.entryCount(); n != 0 { |
| 644 | t.Fatalf("Shutdown left %d entr(ies) behind", n) |
| 645 | } |
| 646 | } |
| 647 | |
| 648 | // execGenBackend wakes through a real ExecBackend's StartGen (a placed flat |
| 649 | // spawn under GenContainment, in a delegated cgroup tree of plain files) |
| 650 | // while the fake keeps the lease, restore and replica. Both kills, the |
| 651 | // spawn path's own (ExecBackend.killGen) and the reaper's (KillGen), answer |
| 652 | // the embedded killGenBackend's err. |
| 653 | type execGenBackend struct { |
| 654 | *killGenBackend |
| 655 | x *ExecBackend |
| 656 | gmu sync.Mutex |
| 657 | gens []uint64 |
| 658 | } |
| 659 | |
| 660 | func (g *execGenBackend) StartGen(ctx context.Context, rt registry.Route, n uint64, perGen bool) (Proc, Gen, error) { |
| 661 | g.gmu.Lock() |
| 662 | g.gens = append(g.gens, n) |
| 663 | g.gmu.Unlock() |
| 664 | return g.x.StartGen(ctx, rt, n, perGen) |
| 665 | } |
| 666 | |
| 667 | func (g *execGenBackend) NextDueVia(ctx context.Context, rt registry.Route, _ string) (int64, error) { |
| 668 | return g.fakeBackend.NextDue(ctx, rt) |
| 669 | } |
| 670 | |
| 671 | func (g *execGenBackend) firstGen() uint64 { |
| 672 | g.gmu.Lock() |
| 673 | defer g.gmu.Unlock() |
| 674 | return g.gens[0] |
| 675 | } |
| 676 | |
| 677 | // A placed flat generation whose health wait fails is stopped by its |
| 678 | // leader only, and a process it forked can live on in g<n> (spec §7.3). |
| 679 | // startChildAt kills g<n> after the leader's stop. When that kill fails, |
| 680 | // the process goes back with an error wrapping errGenUnproven and the wake |
| 681 | // keeps its lease until the sweep proves the child empty. When it |
| 682 | // succeeds, nothing is left and the lease goes back at once. With |
| 683 | // containment off nothing is placed and startChildAt makes no kill. Its |
| 684 | // handle now comes back with the health error (Task 15 fix round 1, ruling |
| 685 | // a), so the failed-wake cleanup's proof asks KillGen once, which answers |
| 686 | // errNoGenContainment as ExecBackend's does with containment off; this |
| 687 | // backend has no host kill, so the lease goes back. |
| 688 | func TestAPlacedSpawnWithAFailedHealthWaitProvesItsChild(t *testing.T) { |
| 689 | old := spawnHealthTimeout |
| 690 | spawnHealthTimeout = 300 * time.Millisecond |
| 691 | t.Cleanup(func() { spawnHealthTimeout = old }) |
| 692 | rows := []struct { |
| 693 | name string |
| 694 | containment bool |
| 695 | killErr error |
| 696 | wantKept bool |
| 697 | }{ |
| 698 | {"placed, kill fails", true, errors.New("g still holds a process"), true}, |
| 699 | {"placed, kill proves the child empty", true, nil, false}, |
| 700 | {"containment off", false, errNoGenContainment, false}, |
| 701 | } |
| 702 | for _, row := range rows { |
| 703 | t.Run(row.name, func(t *testing.T) { |
| 704 | host := "placed.example.com" |
| 705 | x, _ := delegatedBackend(t, host) |
| 706 | cgroup.FakePlacementForTest(t) // a temp dir is no cgroupfs, and a generation is never moved by PID |
| 707 | x.GenContainment = row.containment |
| 708 | x.RunDirOverride = shortRunDir(t) |
| 709 | stageLive(t, x, host, "v1") // exits at once without binding: the health wait fails |
| 710 | a, b, _, _ := testActivator(t, time.Hour) |
| 711 | a.gens = true |
| 712 | kb := &killGenBackend{fakeBackend: b, err: row.killErr} |
| 713 | gb := &execGenBackend{killGenBackend: kb, x: x} |
| 714 | x.killGen = kb.KillGen |
| 715 | a.backend = gb |
| 716 | rt := flatGenRoute(x, host) |
| 717 | |
| 718 | _, err := a.beginRequest(context.Background(), rt) |
| 719 | if err == nil { |
| 720 | t.Fatal("a wake whose health wait failed succeeded") |
| 721 | } |
| 722 | n := gb.firstGen() |
| 723 | a.mu.Lock() |
| 724 | _, kept := a.unprovenWakes[host] |
| 725 | a.mu.Unlock() |
| 726 | if kept != row.wantKept { |
| 727 | t.Fatalf("retained = %v, want %v (err %v)", kept, row.wantKept, err) |
| 728 | } |
| 729 | if !row.containment { |
| 730 | if k := kb.killed(); !slices.Equal(k, []uint64{n}) { |
| 731 | t.Fatalf("KillGen calls = %v with containment off, want only the cleanup's proof of generation %d", k, n) |
| 732 | } |
| 733 | if got := b.releaseCount(); got != 1 { |
| 734 | t.Fatalf("lease releases = %d, want 1", got) |
| 735 | } |
| 736 | return |
| 737 | } |
| 738 | if k := kb.killed(); len(k) == 0 || k[0] != n { |
| 739 | t.Fatalf("KillGen calls = %v, want generation %d's first", k, n) |
| 740 | } |
| 741 | if !row.wantKept { |
| 742 | if got := b.releaseCount(); got != 1 { |
| 743 | t.Fatalf("lease releases = %d after a proven kill, want 1", got) |
| 744 | } |
| 745 | return |
| 746 | } |
| 747 | if got := b.releaseCount(); got != 0 { |
| 748 | t.Fatalf("lease releases = %d while g%d is unproven, want 0", got, n) |
| 749 | } |
| 750 | kb.setErr(nil) |
| 751 | a.sweepOnce() |
| 752 | waitFor(t, "the lease release", func() bool { return b.releaseCount() == 1 }) |
| 753 | }) |
| 754 | } |
| 755 | } |
| 756 | |
| 757 | // unprovenStartBackend is a GenBackend whose next StartGen fails the way a |
| 758 | // generation start fails when its kill cannot be proven (plan 2, |
| 759 | // startUnitAt): fail, while set, answers that one call. Every later call |
| 760 | // spawns through the fake's Start. KillGen comes from killGenBackend. |
| 761 | type unprovenStartBackend struct { |
| 762 | *killGenBackend |
| 763 | smu sync.Mutex |
| 764 | fail func(n uint64) (Proc, error) |
| 765 | gens []uint64 // every generation number StartGen was handed |
| 766 | } |
| 767 | |
| 768 | func (u *unprovenStartBackend) StartGen(ctx context.Context, rt registry.Route, n uint64, _ bool) (Proc, Gen, error) { |
| 769 | u.smu.Lock() |
| 770 | fail := u.fail |
| 771 | u.fail = nil |
| 772 | u.gens = append(u.gens, n) |
| 773 | u.smu.Unlock() |
| 774 | if fail != nil { |
| 775 | p, err := fail(n) |
| 776 | return p, Gen{N: n}, err |
| 777 | } |
| 778 | p, ver, err := u.fakeBackend.Start(ctx, rt) |
| 779 | return p, canonicalGen(rt, n, ver), err |
| 780 | } |
| 781 | |
| 782 | func (u *unprovenStartBackend) NextDueVia(ctx context.Context, rt registry.Route, _ string) (int64, error) { |
| 783 | return u.fakeBackend.NextDue(ctx, rt) |
| 784 | } |
| 785 | |
| 786 | func (u *unprovenStartBackend) firstGen() uint64 { |
| 787 | u.smu.Lock() |
| 788 | defer u.smu.Unlock() |
| 789 | return u.gens[0] |
| 790 | } |
| 791 | |
| 792 | var ( |
| 793 | _ GenBackend = (*unprovenStartBackend)(nil) |
| 794 | _ genKiller = (*unprovenStartBackend)(nil) |
| 795 | ) |
| 796 | |
| 797 | // P2-13 (plan 2's parked L1): a generation start that failed with its kill |
| 798 | // unproven may still be running. The wake keeps the lease (spec §7.4, |
| 799 | // "unproven, the lease is kept"), refuses the host's next wake, and gives |
| 800 | // the lease back only once the sweep proves the generation's child empty |
| 801 | // through KillGen. Before, a start that returned no process released the |
| 802 | // lease at once, and one whose Stop failed released it anyway. |
| 803 | // |
| 804 | // The sentinel rows hold with generations off too: errGenUnproven and |
| 805 | // instancectl.ErrGenUnproven exist only under plan 2's GenContainment, |
| 806 | // which can be on while swaps are off. This is the witness for M9. |
| 807 | func TestSwapWitnessUnprovenWakeKeepsTheLease(t *testing.T) { |
| 808 | t.Parallel() |
| 809 | lost := errors.New("the unit exited before it was observed") |
| 810 | rows := []struct { |
| 811 | name string |
| 812 | gens bool |
| 813 | leader *stubbornProc // the row's process, alive until told to exit; nil: none |
| 814 | err error |
| 815 | }{ |
| 816 | {"errGenUnproven with no process", true, nil, |
| 817 | fmt.Errorf("start unit: %w; %w: kill timed out", lost, errGenUnproven)}, |
| 818 | {"instancectl.ErrGenUnproven with no process", true, nil, |
| 819 | fmt.Errorf("h: start unit: %w", fmt.Errorf("%w: %w", instancectl.ErrGenUnproven, lost))}, |
| 820 | {"a failed stop while the leader lives", true, &stubbornProc{}, |
| 821 | errors.New("health: no answer within 30s")}, |
| 822 | {"generations off, sentinel present", false, nil, |
| 823 | fmt.Errorf("start unit: %w; %w: kill timed out", lost, errGenUnproven)}, |
| 824 | } |
| 825 | for _, row := range rows { |
| 826 | t.Run(row.name, func(t *testing.T) { |
| 827 | a, b, _, _ := testActivator(t, time.Hour) |
| 828 | a.gens = row.gens |
| 829 | kb := &killGenBackend{fakeBackend: b, err: errors.New("the generation's child still holds a process")} |
| 830 | ub := &unprovenStartBackend{killGenBackend: kb, fail: func(uint64) (Proc, error) { |
| 831 | if row.leader != nil { |
| 832 | return row.leader, row.err |
| 833 | } |
| 834 | return nil, row.err |
| 835 | }} |
| 836 | a.backend = ub |
| 837 | host := "unproven.example.com" |
| 838 | rt := testRoute(host) |
| 839 | ctx := context.Background() |
| 840 | retained := func() *unprovenWake { |
| 841 | a.mu.Lock() |
| 842 | defer a.mu.Unlock() |
| 843 | return a.unprovenWakes[host] |
| 844 | } |
| 845 | settled := func() bool { |
| 846 | a.mu.Lock() |
| 847 | defer a.mu.Unlock() |
| 848 | u := a.unprovenWakes[host] |
| 849 | return u == nil || !u.busy |
| 850 | } |
| 851 | |
| 852 | if _, err := a.beginRequest(ctx, rt); err == nil { |
| 853 | t.Fatal("the wake should have failed") |
| 854 | } |
| 855 | if got := b.releaseCount(); got != 0 { |
| 856 | t.Fatalf("lease releases = %d after a failed wake whose generation is unproven, want 0", got) |
| 857 | } |
| 858 | if retained() == nil { |
| 859 | t.Fatal("the failed wake was not retained") |
| 860 | } |
| 861 | n := ub.firstGen() |
| 862 | |
| 863 | // The host's next wake is refused before it takes a lease. |
| 864 | _, err := a.beginRequest(ctx, rt) |
| 865 | if !errors.Is(err, errWakeUnproven) { |
| 866 | t.Fatalf("second wake: err = %v, want errWakeUnproven", err) |
| 867 | } |
| 868 | if acq, _, _ := b.counts(); acq != 1 { |
| 869 | t.Fatalf("Acquire calls = %d, want 1 (the refusal comes before the lease)", acq) |
| 870 | } |
| 871 | |
| 872 | // A sweep whose proof still fails keeps the lease. |
| 873 | a.sweepOnce() |
| 874 | waitFor(t, "the retained wake's retry to finish", settled) |
| 875 | if got := b.releaseCount(); got != 0 || retained() == nil { |
| 876 | t.Fatalf("lease releases = %d after a failed proof, want 0 and the wake retained", got) |
| 877 | } |
| 878 | if row.leader != nil { |
| 879 | if got := row.leader.stops.Load(); got < 2 { |
| 880 | t.Fatalf("leader Stop calls = %d, want the sweep to have stopped it again", got) |
| 881 | } |
| 882 | // The leader exits, but its generation's child is still not |
| 883 | // proven empty: the lease stays. |
| 884 | row.leader.exited.Store(true) |
| 885 | a.sweepOnce() |
| 886 | waitFor(t, "the retained wake's retry to finish", settled) |
| 887 | if got := b.releaseCount(); got != 0 || retained() == nil { |
| 888 | t.Fatalf("lease releases = %d with the leader gone but the child unproven, want 0", got) |
| 889 | } |
| 890 | } |
| 891 | |
| 892 | kb.setErr(nil) |
| 893 | a.sweepOnce() |
| 894 | waitFor(t, "the lease release", func() bool { return b.releaseCount() == 1 }) |
| 895 | waitFor(t, "the retained wake dropped", func() bool { return retained() == nil }) |
| 896 | if k := kb.killed(); !slices.Contains(k, n) { |
| 897 | t.Fatalf("KillGen calls = %v, want generation %d's", k, n) |
| 898 | } |
| 899 | |
| 900 | release, err := a.beginRequest(ctx, rt) |
| 901 | if err != nil { |
| 902 | t.Fatalf("third wake: %v", err) |
| 903 | } |
| 904 | release() |
| 905 | if got := b.startCount(); got != 1 { |
| 906 | t.Fatalf("Start calls = %d, want 1 (the third wake)", got) |
| 907 | } |
| 908 | a.Bounce(host) |
| 909 | waitFor(t, "cleanup", func() bool { return a.entryCount() == 0 }) |
| 910 | }) |
| 911 | } |
| 912 | } |
| 913 | |
| 914 | // syncLog is a log destination that goroutines may write concurrently (the |
| 915 | // reapers log off the test's goroutine). |
| 916 | type syncLog struct { |
| 917 | mu sync.Mutex |
| 918 | b strings.Builder |
| 919 | } |
| 920 | |
| 921 | func (s *syncLog) Write(p []byte) (int, error) { |
| 922 | s.mu.Lock() |
| 923 | defer s.mu.Unlock() |
| 924 | return s.b.Write(p) |
| 925 | } |
| 926 | |
| 927 | func (s *syncLog) String() string { |
| 928 | s.mu.Lock() |
| 929 | defer s.mu.Unlock() |
| 930 | return s.b.String() |
| 931 | } |
| 932 | |
| 933 | // Shutdown covers a retained failed wake (spec §7.4, I2: nothing runs after |
| 934 | // Shutdown returns). No entry waits for one, so Shutdown polls its reaper |
| 935 | // until the generation is proven stopped and the lease given back, or the |
| 936 | // budget runs out, which it logs with the lease still held. Before, Shutdown |
| 937 | // saw no entries and returned at once, leaving the generation running and |
| 938 | // its lease held until the TTL. The retention has no generations gate (the |
| 939 | // sentinel exists only under GenContainment), so neither has the wait. |
| 940 | func TestSwapWitnessShutdownWaitsForAnUnprovenWake(t *testing.T) { |
| 941 | for _, row := range []struct { |
| 942 | name string |
| 943 | gens bool |
| 944 | budget time.Duration |
| 945 | heal bool |
| 946 | }{ |
| 947 | {"healed within the budget", true, 10 * time.Second, true}, |
| 948 | {"healed within the budget, generations off", false, 10 * time.Second, true}, |
| 949 | {"the budget runs out", true, 300 * time.Millisecond, false}, |
| 950 | } { |
| 951 | t.Run(row.name, func(t *testing.T) { |
| 952 | a, b, _, _ := testActivator(t, time.Hour) |
| 953 | a.gens = row.gens |
| 954 | kb := &killGenBackend{fakeBackend: b, err: errors.New("the generation's child still holds a process")} |
| 955 | a.backend = &unprovenStartBackend{killGenBackend: kb, fail: func(uint64) (Proc, error) { |
| 956 | return nil, fmt.Errorf("start unit: health wait failed; %w: kill timed out", errGenUnproven) |
| 957 | }} |
| 958 | host := "unproven.example.com" |
| 959 | retained := func() bool { |
| 960 | a.mu.Lock() |
| 961 | defer a.mu.Unlock() |
| 962 | return a.unprovenWakes[host] != nil |
| 963 | } |
| 964 | if _, err := a.beginRequest(context.Background(), testRoute(host)); err == nil { |
| 965 | t.Fatal("the wake should have failed") |
| 966 | } |
| 967 | if !retained() || b.releaseCount() != 0 { |
| 968 | t.Fatalf("retained %v, lease releases %d: want the failed wake retained with its lease", retained(), b.releaseCount()) |
| 969 | } |
| 970 | logs := &syncLog{} |
| 971 | if !row.heal { |
| 972 | flags := log.Flags() |
| 973 | log.SetOutput(logs) |
| 974 | t.Cleanup(func() { log.SetOutput(os.Stderr); log.SetFlags(flags) }) |
| 975 | } |
| 976 | kills := len(kb.killed()) |
| 977 | |
| 978 | shut := make(chan struct{}) |
| 979 | go func() { |
| 980 | ctx, cancel := context.WithTimeout(context.Background(), row.budget) |
| 981 | defer cancel() |
| 982 | a.Shutdown(ctx) |
| 983 | close(shut) |
| 984 | }() |
| 985 | returned := func() bool { |
| 986 | select { |
| 987 | case <-shut: |
| 988 | return true |
| 989 | default: |
| 990 | return false |
| 991 | } |
| 992 | } |
| 993 | // Two more failed proofs since Shutdown began: it is still |
| 994 | // polling the reaper, so it has not returned while the wake is |
| 995 | // retained. |
| 996 | waitFor(t, "two failed proofs during Shutdown, or its return", func() bool { |
| 997 | return len(kb.killed()) >= kills+2 || returned() |
| 998 | }) |
| 999 | if returned() { |
| 1000 | t.Fatalf("Shutdown returned while the failed wake was retained (KillGen calls during it: %d)", len(kb.killed())-kills) |
| 1001 | } |
| 1002 | |
| 1003 | if row.heal { |
| 1004 | kb.setErr(nil) |
| 1005 | } |
| 1006 | select { |
| 1007 | case <-shut: |
| 1008 | case <-time.After(5 * time.Second): |
| 1009 | t.Fatal("Shutdown did not return") |
| 1010 | } |
| 1011 | if row.heal { |
| 1012 | if retained() || b.releaseCount() != 1 { |
| 1013 | t.Fatalf("after Shutdown: retained %v, lease releases %d; want the wake proven and its lease given back once", retained(), b.releaseCount()) |
| 1014 | } |
| 1015 | return |
| 1016 | } |
| 1017 | if !retained() || b.releaseCount() != 0 { |
| 1018 | t.Fatalf("after the budget ran out: retained %v, lease releases %d; want the lease kept", retained(), b.releaseCount()) |
| 1019 | } |
| 1020 | if got := logs.String(); !strings.Contains(got, host) || !strings.Contains(got, "shutdown") || !strings.Contains(got, "not proven stopped") { |
| 1021 | t.Fatalf("Shutdown's log does not name the retained wake:\n%s", got) |
| 1022 | } |
| 1023 | }) |
| 1024 | } |
| 1025 | } |
| 1026 | |
| 1027 | // sigtermIgnorer is a leader that does not stop when asked: Stop answers |
| 1028 | // ErrStillRunning, as unitProc does when SIGKILL could not prove the unit |
| 1029 | // gone, until the process exits on its own (exit). |
| 1030 | type sigtermIgnorer struct { |
| 1031 | exited atomic.Bool |
| 1032 | stops atomic.Int32 |
| 1033 | } |
| 1034 | |
| 1035 | func (p *sigtermIgnorer) Stop(context.Context) error { |
| 1036 | p.stops.Add(1) |
| 1037 | if p.exited.Load() { |
| 1038 | return nil |
| 1039 | } |
| 1040 | return ErrStillRunning |
| 1041 | } |
| 1042 | func (p *sigtermIgnorer) Exited() bool { return p.exited.Load() } |
| 1043 | |
| 1044 | // fencedStartBackend is a GenBackend whose StartGen has spawned p but |
| 1045 | // returns it only once gate opens: a fence lands between the spawn and the |
| 1046 | // wake's post-spawn check. No KillGen: the leader's exit is the proof. |
| 1047 | type fencedStartBackend struct { |
| 1048 | *fakeBackend |
| 1049 | p *sigtermIgnorer |
| 1050 | entered chan struct{} |
| 1051 | gate chan struct{} |
| 1052 | } |
| 1053 | |
| 1054 | func (s *fencedStartBackend) StartGen(_ context.Context, rt registry.Route, n uint64, _ bool) (Proc, Gen, error) { |
| 1055 | close(s.entered) |
| 1056 | <-s.gate |
| 1057 | return s.p, canonicalGen(rt, n, "v1"), nil |
| 1058 | } |
| 1059 | |
| 1060 | func (s *fencedStartBackend) NextDueVia(context.Context, registry.Route, string) (int64, error) { |
| 1061 | return 0, nil |
| 1062 | } |
| 1063 | |
| 1064 | var _ GenBackend = (*fencedStartBackend)(nil) |
| 1065 | |
| 1066 | // A wake fenced after its spawn (the post-spawn ctx/wakeAllowed check) stops |
| 1067 | // its process. If that process does not stop, the lease is kept until it is |
| 1068 | // proven gone, with generations off too: a fail-closed dormancy exception |
| 1069 | // (plan 3 task 2 fix), because releasing it would let another box take the |
| 1070 | // host while this process still writes. Before, with generations off, |
| 1071 | // cleanupFailedWake released the lease over the live process, as main does. |
| 1072 | // The wake is retained (unprovenWakes): the host's next wake is refused, |
| 1073 | // FenceAndStop reports the stop unproven, and the sweep gives the lease back |
| 1074 | // once the leader has exited. |
| 1075 | func TestSwapWitnessAFencedWakeKeepsItsLeaseWhileItsProcessRuns(t *testing.T) { |
| 1076 | t.Parallel() |
| 1077 | for _, gens := range []bool{false, true} { |
| 1078 | t.Run(fmt.Sprintf("generations %v", gens), func(t *testing.T) { |
| 1079 | a, b, _, _ := testActivator(t, time.Hour) |
| 1080 | a.gens = gens |
| 1081 | p := &sigtermIgnorer{} |
| 1082 | s := &fencedStartBackend{fakeBackend: b, p: p, entered: make(chan struct{}), gate: make(chan struct{})} |
| 1083 | a.backend = s |
| 1084 | host := "fenced.example.com" |
| 1085 | woke := make(chan error, 1) |
| 1086 | go func() { |
| 1087 | ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) |
| 1088 | defer cancel() |
| 1089 | _, err := a.beginRequest(ctx, testRoute(host)) |
| 1090 | woke <- err |
| 1091 | }() |
| 1092 | <-s.entered |
| 1093 | fenced := make(chan error, 1) |
| 1094 | go func() { |
| 1095 | ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) |
| 1096 | defer cancel() |
| 1097 | _, err := a.FenceAndStop(ctx, host, 1) |
| 1098 | fenced <- err |
| 1099 | }() |
| 1100 | waitFor(t, "the fence", func() bool { return !a.wakeAllowed(host) }) |
| 1101 | close(s.gate) |
| 1102 | if err := <-woke; err == nil { |
| 1103 | t.Fatal("the fenced wake should have failed") |
| 1104 | } |
| 1105 | if p.stops.Load() == 0 { |
| 1106 | t.Fatal("the fenced wake did not stop its process") |
| 1107 | } |
| 1108 | if got := b.releaseCount(); got != 0 { |
| 1109 | t.Fatalf("lease releases = %d while the fenced wake's process runs, want 0", got) |
| 1110 | } |
| 1111 | retained := func() bool { |
| 1112 | a.mu.Lock() |
| 1113 | defer a.mu.Unlock() |
| 1114 | return a.unprovenWakes[host] != nil |
| 1115 | } |
| 1116 | settled := func() bool { |
| 1117 | a.mu.Lock() |
| 1118 | defer a.mu.Unlock() |
| 1119 | u := a.unprovenWakes[host] |
| 1120 | return u == nil || !u.busy |
| 1121 | } |
| 1122 | if !retained() { |
| 1123 | t.Fatal("the fenced wake was not retained") |
| 1124 | } |
| 1125 | if err := <-fenced; !errors.Is(err, ErrStopUnproven) { |
| 1126 | t.Fatalf("FenceAndStop = %v, want ErrStopUnproven while the process runs", err) |
| 1127 | } |
| 1128 | |
| 1129 | // A sweep while it still runs keeps the lease. |
| 1130 | a.sweepOnce() |
| 1131 | waitFor(t, "the retained wake's retry", settled) |
| 1132 | if got := b.releaseCount(); got != 0 || !retained() { |
| 1133 | t.Fatalf("lease releases = %d after a sweep with the process alive, want 0 and the wake retained", got) |
| 1134 | } |
| 1135 | |
| 1136 | // It exits; the next sweep proves it and gives the lease back. |
| 1137 | p.exited.Store(true) |
| 1138 | a.sweepOnce() |
| 1139 | waitFor(t, "the lease release", func() bool { return b.releaseCount() == 1 }) |
| 1140 | waitFor(t, "the retained wake dropped", func() bool { return !retained() }) |
| 1141 | a.LiftFence(host, 1) |
| 1142 | }) |
| 1143 | } |
| 1144 | } |
| 1145 | |
| 1146 | // uncontainedStartBackend is the flag-off exec backend's shape for a |
| 1147 | // retained failed wake (Task 15, Astra IMPL-3 F6): KillGen answers |
| 1148 | // errNoGenContainment, as ExecBackend's does with GenContainment off, and |
| 1149 | // KillHost proves carlos-exec@<host> empty, failing while populated. |
| 1150 | type uncontainedStartBackend struct { |
| 1151 | *unprovenStartBackend |
| 1152 | hmu sync.Mutex |
| 1153 | populated bool |
| 1154 | err error // when set, every KillHost answers it |
| 1155 | hostKills int |
| 1156 | } |
| 1157 | |
| 1158 | func (h *uncontainedStartBackend) KillHost(context.Context, registry.Route) error { |
| 1159 | h.hmu.Lock() |
| 1160 | defer h.hmu.Unlock() |
| 1161 | h.hostKills++ |
| 1162 | if h.err != nil { |
| 1163 | return h.err |
| 1164 | } |
| 1165 | if h.populated { |
| 1166 | return errors.New("carlos-exec@<host> still holds a process") |
| 1167 | } |
| 1168 | return nil |
| 1169 | } |
| 1170 | |
| 1171 | func (h *uncontainedStartBackend) setPopulated(on bool) { |
| 1172 | h.hmu.Lock() |
| 1173 | h.populated = on |
| 1174 | h.hmu.Unlock() |
| 1175 | } |
| 1176 | |
| 1177 | func (h *uncontainedStartBackend) hostKillCount() int { |
| 1178 | h.hmu.Lock() |
| 1179 | defer h.hmu.Unlock() |
| 1180 | return h.hostKills |
| 1181 | } |
| 1182 | |
| 1183 | var _ hostKiller = (*uncontainedStartBackend)(nil) |
| 1184 | |
| 1185 | // Task 15, Astra IMPL-3 F6: with the flag off a retained failed wake was |
| 1186 | // proven stopped by its leader's exit alone. KillGen answers |
| 1187 | // errNoGenContainment (there is no g<n>), and genChildEmpty took that as |
| 1188 | // proof, so the lease went back while a process the leader forked could |
| 1189 | // still run in carlos-exec@<host>. The proof now also asks KillHost to |
| 1190 | // empty the host cgroup: while it cannot, the wake stays retained with its |
| 1191 | // lease, and once it can the lease goes back. |
| 1192 | func TestARetainedFlagOffWakeProvesItsHostCgroupEmpty(t *testing.T) { |
| 1193 | t.Parallel() |
| 1194 | a, b, _, _ := testActivator(t, time.Hour) |
| 1195 | a.gens = false |
| 1196 | leader := &sigtermIgnorer{} |
| 1197 | kb := &killGenBackend{fakeBackend: b, err: errNoGenContainment} |
| 1198 | ub := &unprovenStartBackend{killGenBackend: kb, fail: func(uint64) (Proc, error) { |
| 1199 | return leader, errors.New("health: no answer within 30s") |
| 1200 | }} |
| 1201 | hb := &uncontainedStartBackend{unprovenStartBackend: ub, populated: true} |
| 1202 | a.backend = hb |
| 1203 | host := "forked.example.com" |
| 1204 | rt := testRoute(host) |
| 1205 | retained := func() bool { |
| 1206 | a.mu.Lock() |
| 1207 | defer a.mu.Unlock() |
| 1208 | return a.unprovenWakes[host] != nil |
| 1209 | } |
| 1210 | settled := func() bool { |
| 1211 | a.mu.Lock() |
| 1212 | defer a.mu.Unlock() |
| 1213 | u := a.unprovenWakes[host] |
| 1214 | return u == nil || !u.busy |
| 1215 | } |
| 1216 | if _, err := a.beginRequest(context.Background(), rt); err == nil { |
| 1217 | t.Fatal("the wake should have failed") |
| 1218 | } |
| 1219 | if !retained() || b.releaseCount() != 0 { |
| 1220 | t.Fatalf("retained %v, lease releases %d: want the wake whose leader lives retained with its lease", retained(), b.releaseCount()) |
| 1221 | } |
| 1222 | |
| 1223 | // The leader exits, but a descendant it forked is still in the host |
| 1224 | // cgroup: the lease stays and the wake stays retained. |
| 1225 | leader.exited.Store(true) |
| 1226 | a.sweepOnce() |
| 1227 | waitFor(t, "the retained wake's retry", settled) |
| 1228 | if got := b.releaseCount(); got != 0 || !retained() { |
| 1229 | t.Fatalf("lease releases = %d with the leader gone and carlos-exec@<host> populated, want 0 and the wake retained", got) |
| 1230 | } |
| 1231 | if hb.hostKillCount() == 0 { |
| 1232 | t.Fatal("the proof never asked KillHost about the host cgroup") |
| 1233 | } |
| 1234 | if k := kb.killed(); !slices.Contains(k, ub.firstGen()) { |
| 1235 | t.Fatalf("KillGen calls = %v, want generation %d's (answered errNoGenContainment)", k, ub.firstGen()) |
| 1236 | } |
| 1237 | if _, err := a.beginRequest(context.Background(), rt); !errors.Is(err, errWakeUnproven) { |
| 1238 | t.Fatalf("second wake: err = %v, want errWakeUnproven while the host cgroup is unproven", err) |
| 1239 | } |
| 1240 | |
| 1241 | // The host cgroup empties: the next sweep proves the wake stopped. |
| 1242 | hb.setPopulated(false) |
| 1243 | a.sweepOnce() |
| 1244 | waitFor(t, "the lease release", func() bool { return b.releaseCount() == 1 }) |
| 1245 | waitFor(t, "the retained wake dropped", func() bool { return !retained() }) |
| 1246 | } |
| 1247 | |
| 1248 | // The same proof for a flag-off worker's failed wake: it is never numbered |
| 1249 | // with generations off (Gen{}), so genChildEmpty's "no number, nothing to |
| 1250 | // prove" answered for it. A worker runs in carlos-exec@<host> as well, so |
| 1251 | // its leader's exit is not enough either. A backend with no host kill and |
| 1252 | // a contained generation are as before, and so is a box with no delegated |
| 1253 | // cgroup, where the wake ran unaccounted and nothing could be proven. |
| 1254 | func TestFailedWakeStoppedProvesTheHostCgroupWithoutContainment(t *testing.T) { |
| 1255 | t.Parallel() |
| 1256 | exited := &sigtermIgnorer{} |
| 1257 | exited.exited.Store(true) |
| 1258 | for _, row := range []struct { |
| 1259 | name string |
| 1260 | gen Gen |
| 1261 | killGen error // nil: a contained generation whose child KillGen proved empty |
| 1262 | hostKill bool // the backend has KillHost |
| 1263 | populated bool |
| 1264 | hostErr error // what KillHost answers, when set |
| 1265 | want bool |
| 1266 | wantHost bool // KillHost asked |
| 1267 | }{ |
| 1268 | {"uncontained instance, host populated", Gen{N: 4}, errNoGenContainment, true, true, nil, false, true}, |
| 1269 | {"uncontained instance, host empty", Gen{N: 4}, errNoGenContainment, true, false, nil, true, true}, |
| 1270 | {"flag-off worker, host populated", Gen{}, errNoGenContainment, true, true, nil, false, true}, |
| 1271 | {"flag-off worker, host empty", Gen{}, errNoGenContainment, true, false, nil, true, true}, |
| 1272 | {"contained generation", Gen{N: 4}, nil, true, true, nil, true, false}, |
| 1273 | {"no host kill", Gen{N: 4}, errNoGenContainment, false, false, nil, true, false}, |
| 1274 | {"no delegated cgroup", Gen{N: 4}, errNoGenContainment, true, false, cgroup.ErrNoDelegation, true, true}, |
| 1275 | } { |
| 1276 | t.Run(row.name, func(t *testing.T) { |
| 1277 | a, b, _, _ := testActivator(t, time.Hour) |
| 1278 | kb := &killGenBackend{fakeBackend: b, err: row.killGen} |
| 1279 | ub := &unprovenStartBackend{killGenBackend: kb} |
| 1280 | hb := &uncontainedStartBackend{unprovenStartBackend: ub, populated: row.populated, err: row.hostErr} |
| 1281 | if row.hostKill { |
| 1282 | a.backend = hb |
| 1283 | } else { |
| 1284 | a.backend = ub |
| 1285 | } |
| 1286 | if got := a.failedWakeStopped(testRoute("f6.example.com"), row.gen, exited, ""); got != row.want { |
| 1287 | t.Fatalf("failedWakeStopped = %v, want %v", got, row.want) |
| 1288 | } |
| 1289 | if asked := hb.hostKillCount() > 0; asked != row.wantHost { |
| 1290 | t.Fatalf("KillHost asked = %v, want %v", asked, row.wantHost) |
| 1291 | } |
| 1292 | }) |
| 1293 | } |
| 1294 | } |
| 1295 | |
| 1296 | // realFlatBackend is fakeBackend's lease, restore and replica with a real |
| 1297 | // flag-off ExecBackend's spawn (StartGen, startChildAt and its health |
| 1298 | // wait), its KillGen (errNoGenContainment) and its layout. Only KillHost |
| 1299 | // is a fake, since the test box has no delegated cgroup v2: it fails while |
| 1300 | // populated, as a host cgroup that still holds a process does. |
| 1301 | type realFlatBackend struct { |
| 1302 | *fakeBackend |
| 1303 | exec *ExecBackend |
| 1304 | hmu sync.Mutex |
| 1305 | populated bool |
| 1306 | hostKills int |
| 1307 | } |
| 1308 | |
| 1309 | func (r *realFlatBackend) StartGen(ctx context.Context, rt registry.Route, n uint64, perGen bool) (Proc, Gen, error) { |
| 1310 | return r.exec.StartGen(ctx, rt, n, perGen) |
| 1311 | } |
| 1312 | |
| 1313 | func (r *realFlatBackend) NextDueVia(ctx context.Context, rt registry.Route, _ string) (int64, error) { |
| 1314 | return r.fakeBackend.NextDue(ctx, rt) |
| 1315 | } |
| 1316 | |
| 1317 | func (r *realFlatBackend) KillGen(ctx context.Context, rt registry.Route, n uint64) error { |
| 1318 | return r.exec.KillGen(ctx, rt, n) |
| 1319 | } |
| 1320 | |
| 1321 | func (r *realFlatBackend) Layout(rt registry.Route) (Layout, string) { return r.exec.Layout(rt) } |
| 1322 | |
| 1323 | func (r *realFlatBackend) KillHost(context.Context, registry.Route) error { |
| 1324 | r.hmu.Lock() |
| 1325 | defer r.hmu.Unlock() |
| 1326 | r.hostKills++ |
| 1327 | if r.populated { |
| 1328 | return errors.New("carlos-exec@<host> still holds a process") |
| 1329 | } |
| 1330 | return nil |
| 1331 | } |
| 1332 | |
| 1333 | func (r *realFlatBackend) setPopulated(on bool) { |
| 1334 | r.hmu.Lock() |
| 1335 | r.populated = on |
| 1336 | r.hmu.Unlock() |
| 1337 | } |
| 1338 | |
| 1339 | func (r *realFlatBackend) hostKillCount() int { |
| 1340 | r.hmu.Lock() |
| 1341 | defer r.hmu.Unlock() |
| 1342 | return r.hostKills |
| 1343 | } |
| 1344 | |
| 1345 | var ( |
| 1346 | _ GenBackend = (*realFlatBackend)(nil) |
| 1347 | _ genKiller = (*realFlatBackend)(nil) |
| 1348 | _ hostKiller = (*realFlatBackend)(nil) |
| 1349 | _ Classifier = (*realFlatBackend)(nil) |
| 1350 | ) |
| 1351 | |
| 1352 | // Task 15 fix round 1 (rulings a and b), F6 in the shape it really takes: |
| 1353 | // a flat exec child whose health wait times out. startChildAt stopped the |
| 1354 | // leader and returned no handle, so doWake gave the lease back at once, |
| 1355 | // with nothing asked of carlos-exec@<host>, where whatever the app forked |
| 1356 | // still ran. The handle now comes back with the health error, and the |
| 1357 | // failed-wake cleanup proves the host cgroup empty before any release: |
| 1358 | // populated, the wake is retained with its lease; emptied, the sweep gives |
| 1359 | // the lease back. |
| 1360 | func TestARealFlatHealthFailedWakeProvesItsHostCgroup(t *testing.T) { |
| 1361 | requireUnixSockets(t) |
| 1362 | old := spawnHealthTimeout |
| 1363 | spawnHealthTimeout = 500 * time.Millisecond |
| 1364 | t.Cleanup(func() { spawnHealthTimeout = old }) |
| 1365 | a, b, _, _ := testActivator(t, time.Hour) |
| 1366 | a.gens = false |
| 1367 | exec := testExecBackend(t) |
| 1368 | host := "unhealthy-f6.example.com" |
| 1369 | stageServingLive(t, exec, host) |
| 1370 | t.Setenv("ACTIVATOR_FAKE_APP_UNHEALTHY", "1") |
| 1371 | rb := &realFlatBackend{fakeBackend: b, exec: exec, populated: true} |
| 1372 | a.backend = rb |
| 1373 | rt := registry.Route{Host: host, Kind: "instance", Socket: tmpSock(t), DBPath: filepath.Join(t.TempDir(), host+".db"), Hibernate: true} |
| 1374 | if l, _ := exec.Layout(rt); l != LayoutFlat && l != LayoutForeign { |
| 1375 | t.Fatalf("layout = %s, want an exec child's (flat or foreign)", l) |
| 1376 | } |
| 1377 | retained := func() bool { |
| 1378 | a.mu.Lock() |
| 1379 | defer a.mu.Unlock() |
| 1380 | return a.unprovenWakes[host] != nil |
| 1381 | } |
| 1382 | settled := func() bool { |
| 1383 | a.mu.Lock() |
| 1384 | defer a.mu.Unlock() |
| 1385 | u := a.unprovenWakes[host] |
| 1386 | return u == nil || !u.busy |
| 1387 | } |
| 1388 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| 1389 | defer cancel() |
| 1390 | if _, err := a.beginRequest(ctx, rt); err == nil { |
| 1391 | t.Fatal("a never-healthy app woke") |
| 1392 | } |
| 1393 | if got := b.releaseCount(); got != 0 || !retained() { |
| 1394 | t.Fatalf("lease releases = %d, retained %v after a health failure with carlos-exec@<host> populated; want 0 and retained", got, retained()) |
| 1395 | } |
| 1396 | if rb.hostKillCount() == 0 { |
| 1397 | t.Fatal("the failed wake's cleanup never asked KillHost") |
| 1398 | } |
| 1399 | a.sweepOnce() |
| 1400 | waitFor(t, "the retained wake's retry", settled) |
| 1401 | if got := b.releaseCount(); got != 0 || !retained() { |
| 1402 | t.Fatalf("lease releases = %d after a sweep with the host cgroup still populated, want 0", got) |
| 1403 | } |
| 1404 | rb.setPopulated(false) |
| 1405 | a.sweepOnce() |
| 1406 | waitFor(t, "the lease release", func() bool { return b.releaseCount() == 1 }) |
| 1407 | waitFor(t, "the retained wake dropped", func() bool { return !retained() }) |
| 1408 | } |
| 1409 | |
| 1410 | // Ruling b with childProc's semantics and no live leader: a flag-off failed |
| 1411 | // wake whose leader stopped (Stop nil, Exited true, as childProc always |
| 1412 | // answers) still proves the host cgroup before its lease goes. Before, the |
| 1413 | // proof ran only for a leader that lived, so this released at once. |
| 1414 | // Ruling c: a nested row and a unit-backed row have their own proof and are |
| 1415 | // never KillHost'd; their failed wake releases as before. |
| 1416 | func TestAFlagOffFailedWakeProvesTheHostCgroupOnlyForAnExecChild(t *testing.T) { |
| 1417 | t.Parallel() |
| 1418 | for _, row := range []struct { |
| 1419 | name string |
| 1420 | layout Layout |
| 1421 | backing string |
| 1422 | retained bool // populated host cgroup: the wake is kept |
| 1423 | killHost bool |
| 1424 | }{ |
| 1425 | {"flat", LayoutFlat, "", true, true}, |
| 1426 | {"foreign", LayoutForeign, "", true, true}, |
| 1427 | {"nested", LayoutNested, "", false, false}, |
| 1428 | {"unit-backed", LayoutFlat, registry.BackingUnit, false, false}, |
| 1429 | } { |
| 1430 | t.Run(row.name, func(t *testing.T) { |
| 1431 | a, b, _, _ := testActivator(t, time.Hour) |
| 1432 | a.gens = false |
| 1433 | b.layoutFn = func(registry.Route) (Layout, string) { return row.layout, "" } |
| 1434 | leader := &sigtermIgnorer{} |
| 1435 | leader.exited.Store(true) |
| 1436 | kb := &killGenBackend{fakeBackend: b, err: errNoGenContainment} |
| 1437 | ub := &unprovenStartBackend{killGenBackend: kb, fail: func(uint64) (Proc, error) { |
| 1438 | return leader, errors.New("health: no answer within 30s") |
| 1439 | }} |
| 1440 | hb := &uncontainedStartBackend{unprovenStartBackend: ub, populated: true} |
| 1441 | a.backend = hb |
| 1442 | host := "exited-" + row.name + ".example.com" |
| 1443 | rt := testRoute(host) |
| 1444 | rt.Backing = row.backing |
| 1445 | if _, err := a.beginRequest(context.Background(), rt); err == nil { |
| 1446 | t.Fatal("the wake should have failed") |
| 1447 | } |
| 1448 | a.mu.Lock() |
| 1449 | _, retained := a.unprovenWakes[host] |
| 1450 | a.mu.Unlock() |
| 1451 | if retained != row.retained { |
| 1452 | t.Fatalf("retained = %v, want %v", retained, row.retained) |
| 1453 | } |
| 1454 | wantReleases := 1 |
| 1455 | if row.retained { |
| 1456 | wantReleases = 0 |
| 1457 | } |
| 1458 | if got := b.releaseCount(); got != wantReleases { |
| 1459 | t.Fatalf("lease releases = %d, want %d", got, wantReleases) |
| 1460 | } |
| 1461 | if asked := hb.hostKillCount() > 0; asked != row.killHost { |
| 1462 | t.Fatalf("KillHost asked = %v, want %v", asked, row.killHost) |
| 1463 | } |
| 1464 | }) |
| 1465 | } |
| 1466 | } |
| 1467 | |
amadan