amadan
activator: FenceAndStopIfUnfenced installs a fence only when none is held
The layout pass shares the one fence slot per host with the residence move. FenceAndStop replaces a lower generation, so checking Fenced first left a window in which the pass could take over the move's fence and then lift it. The exclusive variant decides under the lock that installs and answers ErrFenceHeld, installing and stopping nothing, when any fence is held.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2 files changed,
+74
−1
internal/activator/maintenance.go+24 −1internal/activator/maintenance_test.go+50 −0
diff --git a/internal/activator/maintenance.go b/internal/activator/maintenance.go| index 1528be129..5b0be53e0 100644 |
| --- a/internal/activator/maintenance.go |
| +++ b/internal/activator/maintenance.go |
| @@ -15,6 +15,7 @@ var ( |
| ErrFenced = errors.New("activator: maintenance fence active") |
| ErrFenceGeneration = errors.New("activator: maintenance generation refused") |
| ErrStopUnproven = errors.New("activator: stop evidence unavailable") |
| + ErrFenceHeld = errors.New("activator: another maintenance fence is held") |
| ) |
| // StopEvidence records local process and replica closure. It grants no lease, |
| @@ -48,6 +49,21 @@ func (o *stopObservation) complete() bool { |
| // ErrStopUnproven guarantees this call installed a fence, but proves no stop. |
| // A missing entry is not evidence that a prior process or replica stopped. |
| func (a *Service) FenceAndStop(ctx context.Context, host string, generation uint64) (*StopEvidence, error) { |
| + return a.fenceAndStop(ctx, host, generation, false) |
| +} |
| + |
| +// FenceAndStopIfUnfenced is FenceAndStop for a caller that must never take |
| +// over somebody else's fence. FenceAndStop replaces a fence at a lower |
| +// generation, and a caller that then lifts its own generation reopens |
| +// admission the other operation closed. Here the check and the install |
| +// happen under the same lock: if any fence is held for host, of any |
| +// generation, it answers ErrFenceHeld, installs nothing and stops nothing. |
| +// Otherwise it is FenceAndStop, answers included. |
| +func (a *Service) FenceAndStopIfUnfenced(ctx context.Context, host string, generation uint64) (*StopEvidence, error) { |
| + return a.fenceAndStop(ctx, host, generation, true) |
| +} |
| + |
| +func (a *Service) fenceAndStop(ctx context.Context, host string, generation uint64, exclusive bool) (*StopEvidence, error) { |
| if a == nil || !controlname.ValidHostShape(host) || generation == 0 { |
| return nil, ErrFenceGeneration |
| } |
| @@ -81,7 +97,14 @@ func (a *Service) FenceAndStop(ctx context.Context, host string, generation uint |
| if a.fences == nil { |
| a.fences = make(map[string]uint64) |
| } |
| - if prior := a.fences[host]; prior > generation { |
| + prior := a.fences[host] |
| + // Only before this call installs: on a later round of the loop the |
| + // fence held is this call's own. |
| + if exclusive && !installed && prior != 0 { |
| + a.mu.Unlock() |
| + return nil, ErrFenceHeld |
| + } |
| + if prior > generation { |
| a.mu.Unlock() |
| return nil, ErrFenceGeneration |
| } |
diff --git a/internal/activator/maintenance_test.go b/internal/activator/maintenance_test.go| index 12ae718df..a90766de7 100644 |
| --- a/internal/activator/maintenance_test.go |
| +++ b/internal/activator/maintenance_test.go |
| @@ -695,3 +695,53 @@ func TestStopUncertainCoversLiveEntriesAndIncompleteHistory(t *testing.T) { |
| t.Fatal("a complete stop observation with no live entry must not read as uncertain") |
| } |
| } |
| + |
| +// TestFenceAndStopIfUnfencedRefusesAnyHeldFence: the layout pass shares the |
| +// one fence slot per host with the residence move, and FenceAndStop would |
| +// replace a lower generation. The exclusive variant decides under the same |
| +// lock that installs, so a fence of any generation, lower, equal or |
| +// higher, refuses it and stays exactly as it was, and nothing is stopped. |
| +func TestFenceAndStopIfUnfencedRefusesAnyHeldFence(t *testing.T) { |
| + for _, held := range []uint64{5, 10, 20} { |
| + t.Run(fmt.Sprint(held), func(t *testing.T) { |
| + a, e, main, _, _ := maintenanceAwake(t) |
| + a.mu.Lock() |
| + a.fences = map[string]uint64{e.host: held} |
| + a.mu.Unlock() |
| + _, err := a.FenceAndStopIfUnfenced(maintenanceContext(t, time.Second), e.host, 10) |
| + if !errors.Is(err, ErrFenceHeld) { |
| + t.Fatalf("err = %v, want ErrFenceHeld", err) |
| + } |
| + a.mu.Lock() |
| + got := a.fences[e.host] |
| + a.mu.Unlock() |
| + if got != held { |
| + t.Fatalf("held fence is %d, want %d", got, held) |
| + } |
| + if main.calls.Load() != 0 || !a.Owns(e.host) { |
| + t.Fatal("a refused exclusive fence stopped the host") |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// TestFenceAndStopIfUnfencedIsFenceAndStopOnAnUnfencedHost: with no fence |
| +// held it installs, stops and proves exactly as FenceAndStop does. |
| +func TestFenceAndStopIfUnfencedIsFenceAndStopOnAnUnfencedHost(t *testing.T) { |
| + a, e, _, _, _ := maintenanceAwake(t) |
| + proof, err := a.FenceAndStopIfUnfenced(maintenanceContext(t, time.Second), e.host, 7) |
| + if err != nil || !proof.Matches(e.host, 7) { |
| + t.Fatalf("exclusive fence on an unfenced awake host: %v", err) |
| + } |
| + if !a.Fenced(e.host) || a.Owns(e.host) { |
| + t.Fatal("not fenced and stopped") |
| + } |
| + // An entry-less host with no stop on record answers ErrStopUnproven |
| + // with its fence installed, as FenceAndStop does. |
| + if _, err := a.FenceAndStopIfUnfenced(maintenanceContext(t, time.Second), "idle.example", 3); !errors.Is(err, ErrStopUnproven) { |
| + t.Fatalf("entry-less host: %v", err) |
| + } |
| + if !a.LiftFence("idle.example", 3) { |
| + t.Fatal("the unproven exclusive fence was not installed at its generation") |
| + } |
| +} |