| index 30582937e..dbf649ac6 100644 |
| --- a/internal/host/restart_test.go |
| +++ b/internal/host/restart_test.go |
| @@ -770,16 +770,121 @@ func sidecarRoute(host, acct, app string) registry.Route { |
| func TestCycleWorkersBouncesAndWakesARunningSidecar(t *testing.T) { |
| routes := []registry.Route{sidecarRoute("bot.x.sidecar.internal", "bdm", "eleven")} |
| moved := [][2]string{{"bdm", "eleven"}} |
| + var mu sync.Mutex |
| var bounced, woken []string |
| - bounce := func(h string) bool { bounced = append(bounced, h); return true } |
| - wake := func(rt registry.Route) error { woken = append(woken, rt.Host); return nil } |
| + bounce := func(h string) bool { mu.Lock(); defer mu.Unlock(); bounced = append(bounced, h); return true } |
| + wake := func(rt registry.Route) error { |
| + mu.Lock() |
| + defer mu.Unlock() |
| + woken = append(woken, rt.Host) |
| + return nil |
| + } |
| n := cycleWorkers(routes, moved, bounce, wake) |
| - if n != 1 || !slices.Equal(bounced, []string{"bot.x.sidecar.internal"}) { |
| - t.Fatalf("bounced %v (n=%d), want exactly the sidecar", bounced, n) |
| + mu.Lock() |
| + bouncedSnap := slices.Clone(bounced) |
| + mu.Unlock() |
| + if n != 1 || !slices.Equal(bouncedSnap, []string{"bot.x.sidecar.internal"}) { |
| + t.Fatalf("bounced %v (n=%d), want exactly the sidecar", bouncedSnap, n) |
| + } |
| + // The wake is dispatched OFF cycleWorkers' return path (fix round 2) — it |
| + // may not have landed yet the instant the call above returns, so this |
| + // waits for it rather than asserting synchronously. |
| + waitForCond(t, "the bounced sidecar to be re-woken", func() bool { |
| + mu.Lock() |
| + defer mu.Unlock() |
| + return slices.Equal(woken, []string{"bot.x.sidecar.internal"}) |
| + }) |
| +} |
| + |
| +// TestCycleWorkersReturnsBeforeASlowWakeCompletes is fix round 2's own |
| +// finding, pinned directly: WakeWorker can block up to 30s per sidecar |
| +// (the settle window), and cycleWorkers' only caller (restartRequestLeg) |
| +// runs synchronously on the box's single per-tick convergence goroutine — |
| +// so cycleWorkers itself must never wait on a wake, or one slow sidecar |
| +// delays the root-restart marker touch and every other app's turn on this |
| +// box's tick. Asserted against a deliberately slow fake wake, the same |
| +// elapsed-time shape Task 8 used for the worker boot pass's own |
| +// bounded-concurrency claim. |
| +func TestCycleWorkersReturnsBeforeASlowWakeCompletes(t *testing.T) { |
| + routes := []registry.Route{sidecarRoute("bot.x.sidecar.internal", "bdm", "eleven")} |
| + moved := [][2]string{{"bdm", "eleven"}} |
| + slow := make(chan struct{}) |
| + bounce := func(h string) bool { return true } |
| + wake := func(rt registry.Route) error { <-slow; return nil } |
| + |
| + start := time.Now() |
| + n := cycleWorkers(routes, moved, bounce, wake) |
| + elapsed := time.Since(start) |
| + close(slow) // release the parked wake goroutine so the test can exit cleanly |
| + |
| + if n != 1 { |
| + t.Fatalf("cycleWorkers = %d, want 1 (the bounce itself is synchronous)", n) |
| + } |
| + if elapsed > time.Second { |
| + t.Fatalf("cycleWorkers took %s to return with a wake blocked indefinitely — the wake must be dispatched asynchronously, not awaited", elapsed) |
| + } |
| +} |
| + |
| +// TestCycleWorkersWakesEveryBouncedSidecarEventually: dispatching wakes |
| +// asynchronously must not mean dropping any of them — every sidecar |
| +// cycleWorkers bounces still gets woken, just not before this function |
| +// returns. |
| +func TestCycleWorkersWakesEveryBouncedSidecarEventually(t *testing.T) { |
| + routes := []registry.Route{ |
| + sidecarRoute("bot.a.sidecar.internal", "bdm", "eleven"), |
| + sidecarRoute("bot.b.sidecar.internal", "bdm", "eleven"), |
| + sidecarRoute("bot.c.sidecar.internal", "bdm", "eleven"), |
| + } |
| + moved := [][2]string{{"bdm", "eleven"}} |
| + var mu sync.Mutex |
| + woken := map[string]bool{} |
| + bounce := func(h string) bool { return true } |
| + wake := func(rt registry.Route) error { |
| + mu.Lock() |
| + defer mu.Unlock() |
| + woken[rt.Host] = true |
| + return nil |
| + } |
| + if n := cycleWorkers(routes, moved, bounce, wake); n != 3 { |
| + t.Fatalf("cycleWorkers = %d, want 3", n) |
| } |
| - if !slices.Equal(woken, []string{"bot.x.sidecar.internal"}) { |
| - t.Fatalf("woken %v, want the bounced sidecar re-woken", woken) |
| + waitForCond(t, "all three sidecars to be re-woken", func() bool { |
| + mu.Lock() |
| + defer mu.Unlock() |
| + return len(woken) == 3 |
| + }) |
| +} |
| + |
| +// TestCycleWorkersFailingWakeDoesNotStopTheOthers: a wake error is logged |
| +// (not asserted here — bounceExecRoutes' own analogous case doesn't pin the |
| +// log line either) and must never prevent another sidecar's wake from |
| +// running, nor cycleWorkers' own return. |
| +func TestCycleWorkersFailingWakeDoesNotStopTheOthers(t *testing.T) { |
| + routes := []registry.Route{ |
| + sidecarRoute("bot.bad.sidecar.internal", "bdm", "eleven"), |
| + sidecarRoute("bot.good.sidecar.internal", "bdm", "eleven"), |
| } |
| + moved := [][2]string{{"bdm", "eleven"}} |
| + var mu sync.Mutex |
| + woken := map[string]bool{} |
| + bounce := func(h string) bool { return true } |
| + wake := func(rt registry.Route) error { |
| + if rt.Host == "bot.bad.sidecar.internal" { |
| + return errors.New("boom") |
| + } |
| + mu.Lock() |
| + defer mu.Unlock() |
| + woken[rt.Host] = true |
| + return nil |
| + } |
| + if n := cycleWorkers(routes, moved, bounce, wake); n != 2 { |
| + t.Fatalf("cycleWorkers = %d, want 2 — both were bounced regardless of the wake outcome", n) |
| + } |
| + waitForCond(t, "the good sidecar to be re-woken despite the other's wake failing", func() bool { |
| + mu.Lock() |
| + defer mu.Unlock() |
| + return woken["bot.good.sidecar.internal"] |
| + }) |
| } |
| |
| // TestCycleWorkersLeavesASleepingSidecarAsleep: Bounce is a no-op on |
amadan