| index 30652589e..0dc9ddc83 100644 |
| --- a/internal/activator/activator_test.go |
| +++ b/internal/activator/activator_test.go |
| @@ -212,6 +212,10 @@ type fakeBackend struct { |
| // yet installed) rather than hoping to hit it. |
| workerStartDelay time.Duration |
| workerEnterN int // StartWorker calls ENTERED (before the delay) |
| + // workerStopGate, when set, is installed on every worker child spawned |
| + // from then on: its Stop blocks until the gate closes, so a test can hold |
| + // a child mid-drain and watch what happens around it. |
| + workerStopGate chan struct{} |
| // order interleaves backend calls and proc stops (start:instance, |
| // start:sidecar, nextDue, stop:sidecar, stop:instance) for the ordering |
| // assertions. |
| @@ -355,7 +359,7 @@ func (b *fakeBackend) StartWorker(ctx context.Context, rt registry.Route, env ma |
| if b.workerErr != nil { |
| return nil, "", b.workerErr |
| } |
| - p := &fakeProc{label: "worker", rec: b, dead: b.workerDeadOnStart} |
| + p := &fakeProc{label: "worker", rec: b, dead: b.workerDeadOnStart, stopGate: b.workerStopGate} |
| b.workers = append(b.workers, p) |
| // Same symlink model as Start: execVerFn, when wired, is the live/<host> |
| // target this exec would have read — so a convergence test can move the |
| @@ -2840,6 +2844,64 @@ func TestShutdownStopsARespawnFromOutlivingTheLease(t *testing.T) { |
| } |
| } |
| |
| +// THE invariant this task has been circling: when respawnDone is observed |
| +// closed, no child spawned by that respawn is still running. |
| +// |
| +// The case that proves it is the raced one — a stop lands after the respawn |
| +// has committed to spawning, so the child it produces must be SHED. If |
| +// respawnDone closed before that shed finished, stop() would fall through and |
| +// reach rep.Close — final sync plus lease release — with a live process still |
| +// pointed at the same database. Narrowing that window is not closing it: the |
| +// shed has to complete first. |
| +func TestRespawnShedsARacedChildBeforeTheLeaseIsReleased(t *testing.T) { |
| + a, b, clk, ver := shippingActivator(t, time.Hour) |
| + if err := a.WakeWorker(alwaysOnWorkerRoute("bot.x.internal")); err != nil { |
| + t.Fatalf("WakeWorker: %v", err) |
| + } |
| + rep := b.lastRep() |
| + // The gate applies to children spawned from now on — i.e. the respawn's, |
| + // the one that will have to be shed. Its drain is held open so the |
| + // ordering is observable rather than a microsecond wide. |
| + gate := make(chan struct{}) |
| + b.mu.Lock() |
| + b.workerStartDelay = 200 * time.Millisecond |
| + b.workerStopGate = gate |
| + b.mu.Unlock() |
| + |
| + ver.Store("v2") |
| + a.sweepOnce() |
| + clk.advance(ConvergeGrace + time.Minute) |
| + a.sweepOnce() |
| + waitFor(t, "the respawn's exec to begin", func() bool { return b.workerEnters() == 2 }) |
| + |
| + // The stop lands mid-exec: past the pre-spawn check, so the child is |
| + // already coming and can only be shed. |
| + if !a.Bounce("bot.x.internal") { |
| + t.Fatal("Bounce found no awake worker to stop") |
| + } |
| + waitFor(t, "the raced child to be spawned", func() bool { return b.workerStarts() == 2 }) |
| + |
| + // It is now draining, held by the gate. The lease must still be held. |
| + deadline := time.Now().Add(300 * time.Millisecond) |
| + for time.Now().Before(deadline) { |
| + if rep.isClosed() { |
| + t.Fatal("the lease was released while the respawn's raced child was still draining — a live process is pointed at a database we no longer hold") |
| + } |
| + time.Sleep(10 * time.Millisecond) |
| + } |
| + |
| + close(gate) // let the shed finish |
| + waitFor(t, "the stop to complete", func() bool { return a.entryCount() == 0 }) |
| + if n := rep.closeCount(); n != 1 { |
| + t.Fatalf("lease closes = %d, want exactly 1", n) |
| + } |
| + for i, p := range b.workers { |
| + if !p.isStopped() { |
| + t.Fatalf("worker child %d survived the stop", i) |
| + } |
| + } |
| +} |
| + |
| // Shutdown CANCELS an in-flight worker spawn rather than waiting it out. The |
| // wait is right everywhere else, but a shutdown answers to a deadline it does |
| // not set: systemd's TimeoutStopSec defaults to 90s and workerSpawnTimeout |
amadan