amadan
Refuse a worker wake once the agent is shutting down
Shutdown sets shuttingDown before it snapshots the entry map, but WakeWorker never read the flag: a wake dispatched after that point — a sidecar-wake leg tick, a boot pass, an operator command landing mid-drain — inserted a fresh entry the snapshot never saw, leaving an orphan child holding the tenant's lease past the agent's own exit with nothing left to stop it. The respawn path already checks the same flag for the same reason; the wake door now does too.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014WvcQVWggXkE5BAkjRDgWF
2 files changed,
+32
−0
internal/activator/activator.go+11 −0internal/activator/activator_test.go+21 −0
diff --git a/internal/activator/activator.go b/internal/activator/activator.go| index 18b67170c..63c18d9af 100644 |
| --- a/internal/activator/activator.go |
| +++ b/internal/activator/activator.go |
| @@ -1216,6 +1216,17 @@ func (a *Service) WakeWorker(rt registry.Route) error { |
| defer deadline.Stop() |
| for { |
| a.mu.Lock() |
| + // Refused once a shutdown has announced itself. Shutdown sets the flag |
| + // BEFORE it snapshots the entry map, so a wake dispatched after that |
| + // point — a sidecar-wake leg tick, a boot pass, an operator command |
| + // landing mid-drain — would insert a fresh entry that snapshot never |
| + // sees: a child holding the tenant's lease past the agent's own exit, |
| + // with nothing left to stop it. The respawn path checks the same flag |
| + // for the same reason. |
| + if a.shuttingDown { |
| + a.mu.Unlock() |
| + return fmt.Errorf("wake worker %s: the agent is shutting down", rt.Host) |
| + } |
| e := a.entries[rt.Host] |
| if e == nil { |
| // Asleep → kick off ONE wake and FALL THROUGH to the switch still |
diff --git a/internal/activator/activator_test.go b/internal/activator/activator_test.go| index fc01cf0e4..37d079dfd 100644 |
| --- a/internal/activator/activator_test.go |
| +++ b/internal/activator/activator_test.go |
| @@ -2456,6 +2456,27 @@ func TestWakeWorkerCoalescesConcurrentCallers(t *testing.T) { |
| } |
| } |
| +// Shutdown announces itself BEFORE it snapshots the entry map, so a wake |
| +// dispatched after that point would insert an entry nothing is left to stop: |
| +// an orphan child holding the tenant's lease past the agent's exit. A wake |
| +// arriving mid-drain is refused outright. |
| +func TestWakeWorkerRefusesOnceShutdownHasBegun(t *testing.T) { |
| + a, b, _, _ := testActivator(t, time.Hour) |
| + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| + defer cancel() |
| + a.Shutdown(ctx) |
| + |
| + if err := a.WakeWorker(testWorkerRoute("bot.x.internal")); err == nil { |
| + t.Fatal("WakeWorker succeeded during shutdown — its child would outlive the agent, holding the lease") |
| + } |
| + if n := b.workerStarts(); n != 0 { |
| + t.Fatalf("workerStarts = %d during shutdown, want 0", n) |
| + } |
| + if a.entryCount() != 0 { |
| + t.Fatal("a wake during shutdown left an entry behind") |
| + } |
| +} |
| + |
| // A failed worker wake releases the lease and leaves nothing behind, exactly |
| // as a failed instance wake does — the error paths that must release a lease |
| // are the reason a worker reuses doWake rather than reimplementing it. |