amadan

carlos / platform Public

zdd plan 3 task 2 fix: a failed wake never releases its lease while its process runs

cleanupFailedWake asked for the termination proof only with generations
on or a containment sentinel. With generations off, a wake fenced or
cancelled after its spawn released its lease even when its process
answered ErrStillRunning or was still alive, as main does: a split-brain
risk, since another box could then wake the host while this process
writes. Now a leader that did not stop keeps the lease whatever the
switch says, and the wake is retained in unprovenWakes (P2-13): the next
wake is refused, FenceAndStop reports the stop unproven, Shutdown waits
for it, and the sweep gives the lease back once genTerminated holds.

A recorded, fail-closed exception to dormancy (controller ruling, task 13
fix round 2). Every other flag-off failed wake is unchanged.

Witness: TestSwapWitnessAFencedWakeKeepsItsLeaseWhileItsProcessRuns.
Found by the task 13 sweep (seeds 194, 2015, 2800).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 5cf536b13bdc5a55aa10b74f5f81092e28312f1f parent c25878d
2 files changed, +130 −3
  • internal/activator/maintenance.go +12 −3
  • internal/activator/opset_test.go +118 −0
diff --git a/internal/activator/maintenance.go b/internal/activator/maintenance.go
index 4d7801e79..949a0b415 100644
--- a/internal/activator/maintenance.go
+++ b/internal/activator/maintenance.go
@@ -256,20 +256,29 @@ func maintenanceResponse(w http.ResponseWriter, r *http.Request) {
// against, which the termination proof needs.
//
// The lease is given back only over a generation proven terminated (spec
-// §7.4) whenever that proof is asked for: with generations on, or when the
+// §7.4) whenever that proof is asked for: with generations on, when the
// start's own error said its kill was unproven (w.genUnproven, which only
-// GenContainment produces and which can be on with generations off).
+// GenContainment produces and which can be on with generations off), or,
+// whatever the switch, when the process did not stop (leaderLives below).
// Unproven, ownership and the lease are kept in unprovenWakes, the sweep
// retries the stop and the proof (reapUnprovenWakesLocked), and no new wake
// of the host starts until it holds.
func (a *Service) cleanupFailedWake(ctx context.Context, rt registry.Route, w *wokenInstance, lease Lease) {
stopCtx, cancel := context.WithTimeout(context.Background(), actStopTimeout)
defer cancel()
+ // leaderLives: the wake's process did not stop when asked (Stop answered
+ // ErrStillRunning) or is still alive. Its lease is kept whatever the
+ // switch says, with generations off too: a fail-closed exception to
+ // dormancy (zero-downtime plan 3, task 2 fix), because giving the lease
+ // back would let another box wake the host while this process still
+ // writes. Every other flag-off failed wake is unchanged.
+ leaderLives := false
if w.proc != nil {
err := w.proc.Stop(stopCtx)
w.stopOutcome.mainStop, w.stopOutcome.mainExited = err == nil, w.proc.Exited()
+ leaderLives = errors.Is(err, ErrStillRunning) || !w.stopOutcome.mainExited
}
- if (a.gens || w.genUnproven) && !a.failedWakeStopped(rt, w.gen, w.proc) {
+ if (a.gens || w.genUnproven || leaderLives) && !a.failedWakeStopped(rt, w.gen, w.proc) {
// Its leader did not exit (ErrStillRunning, a failed kill) or its
// child is not proven empty.
w.stopOutcome.mainExited = false
diff --git a/internal/activator/opset_test.go b/internal/activator/opset_test.go
index 52d19633f..2862871ec 100644
--- a/internal/activator/opset_test.go
+++ b/internal/activator/opset_test.go
@@ -979,3 +979,121 @@ func TestSwapWitnessShutdownWaitsForAnUnprovenWake(t *testing.T) {
})
}
}
+
+// sigtermIgnorer is a leader that does not stop when asked: Stop answers
+// ErrStillRunning, as unitProc does when SIGKILL could not prove the unit
+// gone, until the process exits on its own (exit).
+type sigtermIgnorer struct {
+ exited atomic.Bool
+ stops atomic.Int32
+}
+
+func (p *sigtermIgnorer) Stop(context.Context) error {
+ p.stops.Add(1)
+ if p.exited.Load() {
+ return nil
+ }
+ return ErrStillRunning
+}
+func (p *sigtermIgnorer) Exited() bool { return p.exited.Load() }
+
+// fencedStartBackend is a GenBackend whose StartGen has spawned p but
+// returns it only once gate opens: a fence lands between the spawn and the
+// wake's post-spawn check. No KillGen: the leader's exit is the proof.
+type fencedStartBackend struct {
+ *fakeBackend
+ p *sigtermIgnorer
+ entered chan struct{}
+ gate chan struct{}
+}
+
+func (s *fencedStartBackend) StartGen(_ context.Context, rt registry.Route, n uint64, _ bool) (Proc, Gen, error) {
+ close(s.entered)
+ <-s.gate
+ return s.p, canonicalGen(rt, n, "v1"), nil
+}
+
+func (s *fencedStartBackend) NextDueVia(context.Context, registry.Route, string) (int64, error) {
+ return 0, nil
+}
+
+var _ GenBackend = (*fencedStartBackend)(nil)
+
+// A wake fenced after its spawn (the post-spawn ctx/wakeAllowed check) stops
+// its process. If that process does not stop, the lease is kept until it is
+// proven gone, with generations off too: a fail-closed dormancy exception
+// (plan 3 task 2 fix), because releasing it would let another box take the
+// host while this process still writes. Before, with generations off,
+// cleanupFailedWake released the lease over the live process, as main does.
+// The wake is retained (unprovenWakes): the host's next wake is refused,
+// FenceAndStop reports the stop unproven, and the sweep gives the lease back
+// once the leader has exited.
+func TestSwapWitnessAFencedWakeKeepsItsLeaseWhileItsProcessRuns(t *testing.T) {
+ for _, gens := range []bool{false, true} {
+ t.Run(fmt.Sprintf("generations %v", gens), func(t *testing.T) {
+ a, b, _, _ := testActivator(t, time.Hour)
+ a.gens = gens
+ p := &sigtermIgnorer{}
+ s := &fencedStartBackend{fakeBackend: b, p: p, entered: make(chan struct{}), gate: make(chan struct{})}
+ a.backend = s
+ host := "fenced.example.com"
+ woke := make(chan error, 1)
+ go func() {
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ _, err := a.beginRequest(ctx, testRoute(host))
+ woke <- err
+ }()
+ <-s.entered
+ fenced := make(chan error, 1)
+ go func() {
+ ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+ defer cancel()
+ _, err := a.FenceAndStop(ctx, host, 1)
+ fenced <- err
+ }()
+ waitFor(t, "the fence", func() bool { return !a.wakeAllowed(host) })
+ close(s.gate)
+ if err := <-woke; err == nil {
+ t.Fatal("the fenced wake should have failed")
+ }
+ if p.stops.Load() == 0 {
+ t.Fatal("the fenced wake did not stop its process")
+ }
+ if got := b.releaseCount(); got != 0 {
+ t.Fatalf("lease releases = %d while the fenced wake's process runs, want 0", got)
+ }
+ retained := func() bool {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ return a.unprovenWakes[host] != nil
+ }
+ settled := func() bool {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ u := a.unprovenWakes[host]
+ return u == nil || !u.busy
+ }
+ if !retained() {
+ t.Fatal("the fenced wake was not retained")
+ }
+ if err := <-fenced; !errors.Is(err, ErrStopUnproven) {
+ t.Fatalf("FenceAndStop = %v, want ErrStopUnproven while the process runs", err)
+ }
+
+ // A sweep while it still runs keeps the lease.
+ a.sweepOnce()
+ waitFor(t, "the retained wake's retry", settled)
+ if got := b.releaseCount(); got != 0 || !retained() {
+ t.Fatalf("lease releases = %d after a sweep with the process alive, want 0 and the wake retained", got)
+ }
+
+ // It exits; the next sweep proves it and gives the lease back.
+ p.exited.Store(true)
+ a.sweepOnce()
+ waitFor(t, "the lease release", func() bool { return b.releaseCount() == 1 })
+ waitFor(t, "the retained wake dropped", func() bool { return !retained() })
+ a.LiftFence(host, 1)
+ })
+ }
+}