amadan

carlos / platform Public

layout: a fence kept after a failed handover is the pass's own, and retried (plan 4 final review)

The pass records the generation of each fence it keeps. A later pass that
meets ErrFenceHeld at that exact generation takes the fence as its own,
re-runs the leg, and lifts it once the host stands at the intent. The
activator gains FenceGeneration so the pass can tell its own fence from an
instance move's. The kept-fence reason says the pass holds it and the next
pass retries it.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 006de0e177fb4cdd413173936c6444630620aef3 parent 9459d59
4 files changed, +161 −20
  • internal/activator/maintenance.go +10 −0
  • internal/activator/maintenance_test.go +20 −0
  • internal/layout/pass.go +66 −17
  • internal/layout/pass_linux_test.go +65 −3
diff --git a/internal/activator/maintenance.go b/internal/activator/maintenance.go
index 5b0be53e0..e1c9b2f36 100644
--- a/internal/activator/maintenance.go
+++ b/internal/activator/maintenance.go
@@ -204,6 +204,16 @@ func (a *Service) Fenced(host string) bool {
return a.fences[host] != 0
}
+// FenceGeneration is the generation of the fence installed for host right
+// now, or 0 when there is none. The layout pass uses it to tell a fence it
+// kept itself from another operation's, since both answer ErrFenceHeld to
+// its exclusive install.
+func (a *Service) FenceGeneration(host string) uint64 {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ return a.fences[host]
+}
+
// StopUncertain reports whether host's process state cannot yet be trusted
// as stopped: either an entry for it is still live right now, in any state
// (waking, awake or stopping), or the last recorded stop observation never
diff --git a/internal/activator/maintenance_test.go b/internal/activator/maintenance_test.go
index a90766de7..cd6458d06 100644
--- a/internal/activator/maintenance_test.go
+++ b/internal/activator/maintenance_test.go
@@ -653,6 +653,26 @@ func TestFencedReportsAnInstalledFenceUntilItIsLifted(t *testing.T) {
}
}
+// TestFenceGenerationReportsTheHeldGeneration: the layout pass tells its
+// own kept fence from another operation's by the generation, so the read
+// must say exactly which generation holds, and 0 once none does.
+func TestFenceGenerationReportsTheHeldGeneration(t *testing.T) {
+ a, e, _, _, _ := maintenanceAwake(t)
+ if g := a.FenceGeneration(e.host); g != 0 {
+ t.Fatalf("generation %d before any fence", g)
+ }
+ if _, err := a.FenceAndStop(maintenanceContext(t, time.Second), e.host, 9); err != nil {
+ t.Fatal(err)
+ }
+ if g := a.FenceGeneration(e.host); g != 9 {
+ t.Fatalf("generation = %d, want 9", g)
+ }
+ a.LiftFence(e.host, 9)
+ if g := a.FenceGeneration(e.host); g != 0 {
+ t.Fatalf("generation %d after the lift", g)
+ }
+}
+
// TestStopUncertainCoversLiveEntriesAndIncompleteHistory: a pass deciding
// whether ErrStopUnproven can be treated as a completed stop needs both
// halves of this: an entry that is still live right now (waking, awake or
diff --git a/internal/layout/pass.go b/internal/layout/pass.go
index 0b0fedfce..9eaab8621 100644
--- a/internal/layout/pass.go
+++ b/internal/layout/pass.go
@@ -9,6 +9,7 @@ import (
"path/filepath"
"sort"
"strings"
+ "sync"
"time"
"github.com/carlosframework/platform/internal/activator"
@@ -26,6 +27,7 @@ import (
type Fence interface {
FenceAndStopIfUnfenced(ctx context.Context, host string, generation uint64) (*activator.StopEvidence, error)
LiftFence(host string, generation uint64) bool
+ FenceGeneration(host string) uint64
Quiesced(host string) bool
StopUncertain(host string) bool
}
@@ -71,6 +73,12 @@ type Pass struct {
// not affected, because a host with a journal cannot be woken.
DeferAwake bool
+ // kept is the generation of each fence this pass kept installed after
+ // a failed handover, so a later pass knows that fence as its own
+ // rather than another operation's. Cleared when the fence is lifted.
+ keptMu sync.Mutex
+ kept map[string]uint64
+
// abortAt is a test seam: the step name after which a move returns as
// if the agent had died there, leaving its journal for Recover.
abortAt string
@@ -241,12 +249,12 @@ func (p *Pass) migrateHost(ctx context.Context, host string, routes []registry.R
return Result{host, "refused", fmt.Sprintf("db is missing at %s", db.Src)}
}
}
- gen := p.generation()
- if r, fence := p.quiesce(ctx, host, gen); fence != fenceOurs {
+ r, fence, gen := p.quiesce(ctx, host, p.generation())
+ if fence != fenceOurs {
return r
}
defer p.release(host, gen)
- r, _ := p.shift(host, mode, nil)
+ r, _ = p.shift(host, mode, nil)
return r
}
@@ -255,7 +263,8 @@ type fenceState int
const (
// fenceOurs: this pass installed the fence and the stop is proven or
- // trusted.
+ // trusted, or it is the fence an earlier pass kept after a failed
+ // handover, which has held the host stopped ever since.
fenceOurs fenceState = iota
// fenceHeld: another operation's fence holds the host; the pass
// installed nothing and must lift nothing.
@@ -276,23 +285,30 @@ const (
// entry-less case needs this rather than a proof. Every refusal lifts the
// fence it installed: a fence left behind would keep a host down with
// nothing on its way to release it.
-func (p *Pass) quiesce(ctx context.Context, host string, gen uint64) (Result, fenceState) {
+//
+// It returns the generation the host is fenced at when the fence is the
+// pass's: gen for a fence it just installed, or the generation an earlier
+// pass kept after a failed handover, which the caller lifts in the end.
+func (p *Pass) quiesce(ctx context.Context, host string, gen uint64) (Result, fenceState, uint64) {
if p.Fence == nil {
- return Result{}, fenceOurs
+ return Result{}, fenceOurs, gen
}
fctx, cancel := deadline(ctx)
defer cancel()
_, err := p.Fence.FenceAndStopIfUnfenced(fctx, host, gen)
if err == nil {
- return Result{}, fenceOurs
+ return Result{}, fenceOurs, gen
}
if errors.Is(err, activator.ErrFenceHeld) {
+ if kept, ok := p.keptFence(host); ok {
+ return Result{}, fenceOurs, kept
+ }
p.logf("layout: %s: deferred -> fenced by another operation", host)
- return Result{host, "deferred", "fenced by another operation"}, fenceHeld
+ return Result{host, "deferred", "fenced by another operation"}, fenceHeld, 0
}
unproven := errors.Is(err, activator.ErrStopUnproven)
if unproven && p.SweptAtStart && p.Fence.Quiesced(host) && !p.Fence.StopUncertain(host) {
- return Result{}, fenceOurs
+ return Result{}, fenceOurs, gen
}
p.Fence.LiftFence(host, gen)
reason := "stop unproven"
@@ -300,13 +316,46 @@ func (p *Pass) quiesce(ctx context.Context, host string, gen uint64) (Result, fe
reason += ": " + err.Error()
}
p.logf("layout: %s: deferred -> %s", host, reason)
- return Result{host, "deferred", reason}, fenceNone
+ return Result{host, "deferred", reason}, fenceNone, 0
+}
+
+// keep records a fence this pass leaves installed after a failed handover.
+func (p *Pass) keep(host string, gen uint64) {
+ p.keptMu.Lock()
+ defer p.keptMu.Unlock()
+ if p.kept == nil {
+ p.kept = map[string]uint64{}
+ }
+ p.kept[host] = gen
+}
+
+// keptFence reports the generation of a fence this pass kept for host,
+// provided the fence installed now is still exactly that one. A record
+// the activator no longer matches is dropped, so a later fence of another
+// operation is never taken for the pass's own.
+func (p *Pass) keptFence(host string) (uint64, bool) {
+ p.keptMu.Lock()
+ defer p.keptMu.Unlock()
+ gen, ok := p.kept[host]
+ if !ok {
+ return 0, false
+ }
+ if p.Fence.FenceGeneration(host) != gen {
+ delete(p.kept, host)
+ return 0, false
+ }
+ return gen, true
}
func (p *Pass) release(host string, gen uint64) {
if p.Fence != nil {
p.Fence.LiftFence(host, gen)
}
+ p.keptMu.Lock()
+ defer p.keptMu.Unlock()
+ if p.kept[host] == gen {
+ delete(p.kept, host)
+ }
}
// shift is steps 3 to 9 for a host this pass already holds fenced and
@@ -655,9 +704,8 @@ func (p *Pass) recoverHost(ctx context.Context, host string, byHost map[string]r
if err != nil {
return Result{host, "refused", notActing(err)}
}
- gen := p.generation()
keepFence := false
- r, fence := p.quiesce(ctx, host, gen)
+ r, fence, gen := p.quiesce(ctx, host, p.generation())
switch fence {
case fenceNone:
return r
@@ -732,12 +780,13 @@ func (p *Pass) recoverHost(ctx context.Context, host string, byHost map[string]r
return Result{host, "recovered", fmt.Sprintf("completed to %s, then converged to %s", j.Mode, intent)}
case h == handoverFailed:
// Nothing on disk is known to fence the host, so the pass's own
- // fence stays and the host stays down until an operator or an
- // agent restart acts. Releasing it here would run it at the
- // completed endpoint, which after a forged revert is a nested
- // host running flat.
+ // fence stays and the host stays down. Releasing it here would run
+ // it at the completed endpoint, which after a forged revert is a
+ // nested host running flat. The generation is recorded so the next
+ // pass knows the fence as its own and runs the leg again.
keepFence = true
- reason := fmt.Sprintf("completed to %s; handing over to the move to %s failed: %s; fence kept", j.Mode, intent, next.Reason)
+ p.keep(host, gen)
+ reason := fmt.Sprintf("completed to %s; handing over to the move to %s failed: %s. the fence is held by this pass after a failed handover; the next pass retries it", j.Mode, intent, next.Reason)
p.logf("layout: %s: refused -> %s", host, reason)
return Result{host, "refused", reason}
case h == handoverNone:
diff --git a/internal/layout/pass_linux_test.go b/internal/layout/pass_linux_test.go
index a82dd1354..43b8e5692 100644
--- a/internal/layout/pass_linux_test.go
+++ b/internal/layout/pass_linux_test.go
@@ -69,8 +69,9 @@ func (f *fakeFence) LiftFence(h string, gen uint64) bool {
return true
}
-func (f *fakeFence) Quiesced(h string) bool { return !f.awake[h] }
-func (f *fakeFence) StopUncertain(h string) bool { return f.uncertain[h] }
+func (f *fakeFence) FenceGeneration(h string) uint64 { return f.fences[h] }
+func (f *fakeFence) Quiesced(h string) bool { return !f.awake[h] }
+func (f *fakeFence) StopUncertain(h string) bool { return f.uncertain[h] }
func openLayoutRegistry(t *testing.T, dir string) *registry.Registry {
t.Helper()
@@ -1209,7 +1210,7 @@ func TestRecoverKeepsTheFenceWhenTheHandoverFails(t *testing.T) {
}
f.events = nil
r := byHost(t, p.Recover(context.Background(), ModeNested))[host]
- if r.Outcome != "refused" || !strings.Contains(r.Reason, "fence kept") {
+ if r.Outcome != "refused" || !strings.Contains(r.Reason, "held by this pass after a failed handover") {
t.Fatalf("result = %+v", r)
}
j, found, err := ReadJournal(p.DataDir, host)
@@ -1229,6 +1230,67 @@ func TestRecoverKeepsTheFenceWhenTheHandoverFails(t *testing.T) {
}
}
+// A fence the pass kept after a failed handover is its own, not another
+// operation's. The next pass recognises it by its generation, re-runs the
+// leg once <data>/layout is writable again, and lifts it when the host
+// stands at the intent.
+func TestRecoverRetriesAHandoverUnderTheFenceItKept(t *testing.T) {
+ if os.Geteuid() == 0 {
+ t.Skip("root ignores the directory mode this test relies on")
+ }
+ p, f := box(t)
+ crashedRevert(t, p)
+ dir := journalDir(p.DataDir)
+ calls := 0
+ p.Serialize = func(fn func()) {
+ calls++
+ if calls == 2 {
+ if err := os.Chmod(dir, 0o500); err != nil {
+ t.Fatal(err)
+ }
+ defer os.Chmod(dir, 0o700)
+ }
+ fn()
+ }
+ first := byHost(t, p.Recover(context.Background(), ModeNested))[host]
+ if first.Outcome != "refused" || !strings.Contains(first.Reason, "held by this pass after a failed handover") || !strings.Contains(first.Reason, "the next pass retries it") {
+ t.Fatalf("first pass: %+v", first)
+ }
+ kept := f.fences[host]
+ if kept == 0 {
+ t.Fatal("the fence was not kept")
+ }
+ p.Serialize = nil
+ f.events = nil
+ res := p.Recover(context.Background(), ModeNested)
+ second := byHost(t, res)[host]
+ if second.Outcome != "recovered" || !strings.Contains(second.Reason, "converged to nested") {
+ t.Fatalf("second pass: %+v", second)
+ }
+ for _, r := range append(res, first) {
+ if strings.Contains(r.Reason, "another operation") {
+ t.Errorf("the pass's own fence reported as another operation's: %+v", r)
+ }
+ }
+ if JournalPresent(p.DataDir, host) || rowAt(t, p, host) != endpoints(t, p, host, ModeNested) {
+ t.Fatalf("host not converged: journal %v, row %+v", JournalPresent(p.DataDir, host), rowAt(t, p, host))
+ }
+ if f.fences[host] != 0 {
+ t.Fatalf("the kept fence (generation %d) is still installed at %d", kept, f.fences[host])
+ }
+ if !slicesContains(f.events, "lift "+host) || slicesContains(f.events, "fence "+host) {
+ t.Errorf("events = %v, want the kept fence lifted and no new one installed", f.events)
+ }
+ // The record went with the fence: a later foreign fence at the same
+ // generation is not mistaken for the pass's own.
+ p.keptMu.Lock()
+ _, still := p.kept[host]
+ p.keptMu.Unlock()
+ if still {
+ t.Error("the kept generation is still recorded after the lift")
+ }
+}
+
// A successful handover leaves exactly one journal, the converge leg's
// own, and no temp file.
func TestRecoverHandsOverToTheNewJournalInOneStep(t *testing.T) {