amadan
zdd-5 task 14: a crash inside ReplaceWorker's observation is coalesced
Pins Task 11's carry: with CoalesceRecycle answering true while the attempt runs, the sweep's crash restart does not respawn the attempt's build behind it, and the observation falls back to the good build.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
1 file changed,
+50
−0
internal/activator/workerswap_test.go+50 −0
diff --git a/internal/activator/workerswap_test.go b/internal/activator/workerswap_test.go| index 5c022fde8..6849b5c6e 100644 |
| --- a/internal/activator/workerswap_test.go |
| +++ b/internal/activator/workerswap_test.go |
| @@ -86,6 +86,56 @@ func TestReplaceWorkerFallsBackToTheGoodBuild(t *testing.T) { |
| } |
| } |
| +// Carried to Task 14 (from Task 11): a worker crash inside ReplaceWorker's |
| +// observation is the attempt's to read. The sweep sees the dead child first |
| +// and asks CoalesceRecycle, which the scheduler answers true while the |
| +// host's attempt runs (rollout.Manager.CoalesceRecycle). So the sweep's |
| +// crash restart does not respawn the attempt's build behind it, and the |
| +// observation reads the crash as the build's failure and falls back to the |
| +// good build. Without the hook the sweep respawns v2 and the attempt never |
| +// sees its child fail. |
| +func TestACrashInsideAReplaceWorkerObservationIsCoalesced(t *testing.T) { |
| + // A 2 s window polls every 250 ms (settlePoll), so the sweep below runs |
| + // well before the observation's next look at the child. |
| + shrinkFirstWake(t, 55*time.Second, 25*time.Second, 2*time.Second) |
| + a, b, _, _, _ := stagedActivator(t) |
| + host := "bot.x.internal" |
| + var running atomic.Bool |
| + var asked atomic.Int32 |
| + a.CoalesceRecycle = func(h string) bool { |
| + if h == host { |
| + asked.Add(1) |
| + } |
| + return running.Load() |
| + } |
| + if err := a.WakeWorker(alwaysOnWorkerRoute(host)); err != nil { |
| + t.Fatal(err) |
| + } |
| + good := fakeIdentity("v1") |
| + go func() { |
| + waitForWorkerStarts(t, b, 2) |
| + time.Sleep(100 * time.Millisecond) |
| + b.lastWorker().crash() |
| + a.sweepOnce() // the crash restart's arm, inside the observation |
| + }() |
| + running.Store(true) |
| + res := a.ReplaceWorker(context.Background(), workerAttempt(CauseRelease, ModeOverlap, fakeIdentity("v2")), &good, one) |
| + running.Store(false) |
| + if asked.Load() == 0 { |
| + t.Fatal("the sweep never asked CoalesceRecycle about the crashed child") |
| + } |
| + if res.Outcome != SwapFailed || res.Gen.Build != "v1" { |
| + t.Fatalf("ReplaceWorker = %+v, want failed with the good v1 serving", res) |
| + } |
| + calls := b.calls() |
| + if len(calls) != 3 || *calls[1] != fakeIdentity("v2") || *calls[2] != good { |
| + t.Fatalf("spawns %v, want the wake, v2 once, then the good build: the sweep respawned behind the attempt", calls) |
| + } |
| + if got := a.entryServing(host); got != good { |
| + t.Fatalf("serving %+v, want the good build", got) |
| + } |
| +} |
| + |
| // Spec §6.4, §8.3: a hold build that ran never falls back. The instance |
| // stays down. |
| func TestReplaceWorkerInHoldModeStaysDown(t *testing.T) { |