amadan

carlos / platform Public

layout: fence exclusively, recover under a foreign fence, gate the nested leg

The pass installs its fence with FenceAndStopIfUnfenced and defers a host
another operation holds, with no window between the check and the
install. Recovery on such a host completes the journal under that fence,
installing and lifting nothing, and only for a host with no live entry;
unless the completed endpoint is the intent it keeps the journal at done
so the host stays fenced if the other operation lets go. Completing a
journal stays ungated, but the leg from a completed endpoint toward a
nested intent asks the isolation gate, and a refused host is released at
its flat endpoint.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude ff309e91dbf336afe80b864f3baf92c9c8521b95 parent cf08b41
2 files changed, +259 −44
  • internal/layout/pass.go +103 −36
  • internal/layout/pass_linux_test.go +156 −8

CI passed — run details

diff --git a/internal/layout/pass.go b/internal/layout/pass.go
index fae2240b2..3d9eaed8c 100644
--- a/internal/layout/pass.go
+++ b/internal/layout/pass.go
@@ -19,14 +19,13 @@ import (
)
// Fence is what the pass needs from the activator (design §8.2 step 2):
-// stop the host and refuse wakes for it (FenceAndStop), let it wake again
-// (LiftFence, at exactly the generation this pass installed), and the three
-// reads that decide whether an unproven stop can be trusted and whether
-// somebody else already holds the host's one fence slot.
+// stop the host and refuse wakes for it, but only when no other operation
+// already holds the host's one fence slot (FenceAndStopIfUnfenced); let it
+// wake again (LiftFence, at exactly the generation this pass installed);
+// and the two reads that decide whether an unproven stop can be trusted.
type Fence interface {
- FenceAndStop(ctx context.Context, host string, generation uint64) (*activator.StopEvidence, error)
+ FenceAndStopIfUnfenced(ctx context.Context, host string, generation uint64) (*activator.StopEvidence, error)
LiftFence(host string, generation uint64) bool
- Fenced(host string) bool
Quiesced(host string) bool
StopUncertain(host string) bool
}
@@ -218,41 +217,57 @@ func (p *Pass) migrateHost(ctx context.Context, host string, routes []registry.R
if err := Preflight(plan, routes); err != nil {
return Result{host, "refused", err.Error()}
}
- // The activator has one fence slot per host, and FenceAndStop replaces
- // a lower generation. Fencing over another operation's fence and then
- // lifting ours would reopen admission that operation closed.
- if p.Fence != nil && p.Fence.Fenced(host) {
- return Result{host, "deferred", "fenced by another operation"}
- }
gen := p.generation()
- if r, ok := p.quiesce(ctx, host, gen); !ok {
+ if r, fence := p.quiesce(ctx, host, gen); fence != fenceOurs {
return r
}
defer p.release(host, gen)
return p.shift(host, mode)
}
-// quiesce installs this pass's fence and decides whether the host is
-// proven stopped. An unproven stop is trusted only when nothing can be
+// fenceState is who holds a host's fence once quiesce returns.
+type fenceState int
+
+const (
+ // fenceOurs: this pass installed the fence and the stop is proven or
+ // trusted.
+ fenceOurs fenceState = iota
+ // fenceHeld: another operation's fence holds the host; the pass
+ // installed nothing and must lift nothing.
+ fenceHeld
+ // fenceNone: the pass installed nothing, or lifted what it installed,
+ // and the host is not to be touched.
+ fenceNone
+)
+
+// quiesce installs this pass's fence, unless another operation's fence
+// already holds the host, and decides whether the host is proven stopped.
+// The residence move shares the activator's one fence slot per host, so
+// installing over its fence and later lifting ours would reopen admission
+// the move closed; the exclusive install refuses instead, atomically. An unproven stop is trusted only when nothing can be
// running: no live entry, no stop on record that failed to finish, and a
// startup sweep that stopped anything an earlier agent left. After an
// agent restart no asleep host has a stop on record, which is why the
// entry-less case needs this rather than a proof. Every refusal lifts the
// fence it installed: a fence left behind would keep a host down with
// nothing on its way to release it.
-func (p *Pass) quiesce(ctx context.Context, host string, gen uint64) (Result, bool) {
+func (p *Pass) quiesce(ctx context.Context, host string, gen uint64) (Result, fenceState) {
if p.Fence == nil {
- return Result{}, true
+ return Result{}, fenceOurs
}
fctx, cancel := deadline(ctx)
defer cancel()
- _, err := p.Fence.FenceAndStop(fctx, host, gen)
+ _, err := p.Fence.FenceAndStopIfUnfenced(fctx, host, gen)
if err == nil {
- return Result{}, true
+ return Result{}, fenceOurs
+ }
+ if errors.Is(err, activator.ErrFenceHeld) {
+ p.logf("layout: %s: deferred -> fenced by another operation", host)
+ return Result{host, "deferred", "fenced by another operation"}, fenceHeld
}
unproven := errors.Is(err, activator.ErrStopUnproven)
if unproven && p.SweptAtStart && p.Fence.Quiesced(host) && !p.Fence.StopUncertain(host) {
- return Result{}, true
+ return Result{}, fenceOurs
}
p.Fence.LiftFence(host, gen)
reason := "stop unproven"
@@ -260,7 +275,7 @@ func (p *Pass) quiesce(ctx context.Context, host string, gen uint64) (Result, bo
reason += ": " + err.Error()
}
p.logf("layout: %s: deferred -> %s", host, reason)
- return Result{host, "deferred", reason}, false
+ return Result{host, "deferred", reason}, fenceNone
}
func (p *Pass) release(host string, gen uint64) {
@@ -468,11 +483,12 @@ func (p *Pass) Recover(ctx context.Context, intent Mode) []Result {
byHost[rt.Host] = rt
}
var out []Result
+ gate := p.nestedGate()
for _, host := range names {
if ctx.Err() != nil {
break
}
- r := p.recoverHost(ctx, host, byHost, intent)
+ r := p.recoverHost(ctx, host, byHost, intent, gate)
if r.Outcome == "refused" {
p.logf("layout: %s: recovery refused -> %s", host, r.Reason)
}
@@ -481,6 +497,22 @@ func (p *Pass) Recover(ctx context.Context, intent Mode) []Result {
return out
}
+// nestedGate asks NestedOK at most once, on first use, and remembers the
+// answer for the rest of one recovery, as Converge asks it once per pass.
+func (p *Pass) nestedGate() func() error {
+ asked := false
+ var answer error
+ return func() error {
+ if p.NestedOK == nil {
+ return nil
+ }
+ if !asked {
+ asked, answer = true, p.NestedOK()
+ }
+ return answer
+ }
+}
+
// notActing makes a recovery refusal say, in the words design §8.2 uses,
// that the pass left the host alone and fenced.
func notActing(err error) string {
@@ -495,7 +527,22 @@ func notActing(err error) string {
// under that fence, converges the host to intent. Releasing the host
// between the two would let a forged or stale journal run it for a moment
// in a layout the fleet did not ask for (design §3.3).
-func (p *Pass) recoverHost(ctx context.Context, host string, byHost map[string]registry.Route, intent Mode) Result {
+//
+// Completing a journal is never gated on the isolation probe; the leg
+// after it toward a nested intent is, exactly as Converge(ModeNested) is.
+// A host the gate stops there is released at its completed endpoint, which
+// is then flat and safe to serve.
+//
+// When another operation's fence already holds the host (the residence
+// move's cut), the pass completes the journal under that fence, installing
+// and lifting nothing, provided the host has no live entry: the move's own
+// fence proved or is proving the stop, and the move refuses a host while a
+// journal exists, so the journal has to be finished for the move to go on.
+// Without a fence of its own the pass cannot converge the host, so unless
+// the completed endpoint is already the intent it keeps the journal at
+// done. The journal then keeps the host fenced if the move aborts and
+// lifts its fence, and the next recovery converges it.
+func (p *Pass) recoverHost(ctx context.Context, host string, byHost map[string]registry.Route, intent Mode, gate func() error) Result {
if !controlname.ValidHostShape(host) {
return Result{host, "refused", "journal file name is not a host; not acting"}
}
@@ -520,10 +567,17 @@ func (p *Pass) recoverHost(ctx context.Context, host string, byHost map[string]r
return Result{host, "refused", notActing(err)}
}
gen := p.generation()
- if r, ok := p.quiesce(ctx, host, gen); !ok {
+ r, fence := p.quiesce(ctx, host, gen)
+ switch fence {
+ case fenceNone:
return r
+ case fenceHeld:
+ if !p.Fence.Quiesced(host) {
+ return Result{host, "deferred", "fenced by another operation and not yet stopped"}
+ }
+ case fenceOurs:
+ defer p.release(host, gen)
}
- defer p.release(host, gen)
// The row was read before the host stopped; check the journal against
// the row as it stands now that nothing else can wake it.
fresh, err := p.Registry.Get(host)
@@ -537,20 +591,32 @@ func (p *Pass) recoverHost(ctx context.Context, host string, byHost map[string]r
if err != nil {
return Result{host, "refused", notActing(err)}
}
- // With the intent unreadable, a completed revert stays fenced at done:
- // a flat tenant can forge a valid revert journal for a nested host, and
- // releasing it flat would run it unisolated until an intent is read.
- hold := intent == IntentUnreadable && j.Mode == ModeFlat
+ var holdReason string
+ switch {
+ case fence == fenceHeld && intent != j.Mode:
+ holdReason = fmt.Sprintf("completed to %s under another operation's fence; journal kept at done until that fence is lifted", j.Mode)
+ case intent == IntentUnreadable && j.Mode == ModeFlat:
+ // A flat tenant can forge a valid revert journal for a nested
+ // host, and releasing it flat would run it unisolated until an
+ // intent is read.
+ holdReason = "completed to flat; journal kept at done until the layout intent can be read"
+ }
var res Result
- p.serialize(func() { res = p.complete(plan, j, atFrom, hold) })
+ p.serialize(func() { res = p.complete(plan, j, atFrom, holdReason) })
if res.Outcome == "recovered" || res.Outcome == "deferred" {
// Both flipped the row (or found it flipped); a held host is
// still fenced by its journal, but routing should say where it is.
p.reload()
}
- if res.Outcome != "recovered" || intent == "" || intent == IntentUnreadable || intent == j.Mode {
+ if res.Outcome != "recovered" || fence == fenceHeld || intent == "" || intent == IntentUnreadable || intent == j.Mode {
return res
}
+ if intent == ModeNested {
+ if err := gate(); err != nil {
+ p.logf("layout: %s: released -> %s; not converging to nested: isolation not ready: %v", host, j.Mode, err)
+ return Result{host, "recovered", fmt.Sprintf("completed to %s; not converging to nested: isolation not ready: %v", j.Mode, err)}
+ }
+ }
next := p.shift(host, intent)
switch next.Outcome {
case "migrated", "reverted":
@@ -565,8 +631,9 @@ func (p *Pass) recoverHost(ctx context.Context, host string, byHost map[string]r
// list). A done journal whose row is still at from is refused rather than
// removed: a crash cannot produce it, since done is written only after the
// row's swap committed, and removing it would release a row that names the
-// files' old place.
-func (p *Pass) complete(plan *Plan, j *Journal, atFrom, hold bool) Result {
+// files' old place. A non-empty holdReason completes everything but the
+// journal's removal and reports the host deferred with that reason.
+func (p *Pass) complete(plan *Plan, j *Journal, atFrom bool, holdReason string) Result {
host, phase := plan.Host, j.Phase
switch j.Phase {
case PhaseFiles:
@@ -593,9 +660,9 @@ func (p *Pass) complete(plan *Plan, j *Journal, atFrom, hold bool) Result {
default:
return Result{host, "refused", fmt.Sprintf("journal phase %q unknown; not acting", j.Phase)}
}
- if hold {
- p.logf("layout: %s: completed -> %s; journal kept at done until the layout intent can be read", host, j.Mode)
- return Result{host, "deferred", "completed to flat; journal kept until the layout intent can be read"}
+ if holdReason != "" {
+ p.logf("layout: %s: held -> %s", host, holdReason)
+ return Result{host, "deferred", holdReason}
}
if err := RemoveJournal(p.DataDir, host); err != nil {
return Result{host, "refused", err.Error()}
diff --git a/internal/layout/pass_linux_test.go b/internal/layout/pass_linux_test.go
index 2f9b6dff7..812df8981 100644
--- a/internal/layout/pass_linux_test.go
+++ b/internal/layout/pass_linux_test.go
@@ -25,11 +25,13 @@ const (
replica = "r.example.com"
)
-// fakeFence is the activator's fence as the pass sees it. By default every
-// stop is proven; unproven, awake and uncertain make FenceAndStop answer
-// ErrStopUnproven and shape what Quiesced and StopUncertain say after it.
-// events records every fence and lift, and box's Reload adds "reload", so a
-// test can read the order the pass did things in.
+// fakeFence is the activator's fence as the pass sees it. A fence already
+// in fences, of any generation, refuses a new one with ErrFenceHeld, as the
+// real exclusive install does. Otherwise every stop is proven; unproven,
+// awake and uncertain make it answer ErrStopUnproven and shape what
+// Quiesced and StopUncertain say after it. events records every fence,
+// refusal and lift, and box's Reload adds "reload", so a test can read the
+// order the pass did things in.
type fakeFence struct {
fences map[string]uint64
unproven map[string]bool
@@ -39,10 +41,14 @@ type fakeFence struct {
events []string
}
-func (f *fakeFence) FenceAndStop(_ context.Context, h string, gen uint64) (*activator.StopEvidence, error) {
+func (f *fakeFence) FenceAndStopIfUnfenced(_ context.Context, h string, gen uint64) (*activator.StopEvidence, error) {
if f.fences == nil {
f.fences = map[string]uint64{}
}
+ if f.fences[h] != 0 {
+ f.events = append(f.events, "held "+h)
+ return nil, activator.ErrFenceHeld
+ }
f.fences[h] = gen
f.events = append(f.events, "fence "+h)
if f.onFence != nil {
@@ -63,7 +69,6 @@ func (f *fakeFence) LiftFence(h string, gen uint64) bool {
return true
}
-func (f *fakeFence) Fenced(h string) bool { return f.fences[h] != 0 }
func (f *fakeFence) Quiesced(h string) bool { return !f.awake[h] }
func (f *fakeFence) StopUncertain(h string) bool { return f.uncertain[h] }
@@ -411,7 +416,7 @@ func TestConvergeDefersAHostSomeoneElseFenced(t *testing.T) {
}
p.Generation = func() uint64 { return 1000 }
res := byHost(t, p.Converge(context.Background(), ModeNested))
- if res[host].Outcome != "deferred" {
+ if res[host].Outcome != "deferred" || !strings.Contains(res[host].Reason, "fenced by another operation") {
t.Fatalf("fenced host: %+v", res[host])
}
if res[replica].Outcome != "migrated" {
@@ -755,3 +760,146 @@ func TestRecoverHoldsACompletedRevertWhileTheIntentIsUnreadable(t *testing.T) {
}
}
}
+
+// abortedNestedMove leaves both hosts with a nested journal at phase files
+// and their files already at the nested destination.
+func abortedNestedMove(t *testing.T, p *Pass) {
+ t.Helper()
+ p.abortAt = "after-files"
+ p.Converge(context.Background(), ModeNested)
+ p.abortAt = ""
+ for _, h := range []string{host, replica} {
+ if j, found, err := ReadJournal(p.DataDir, h); err != nil || !found || j.Phase != PhaseFiles {
+ t.Fatalf("setup %s: %+v %v %v", h, j, found, err)
+ }
+ }
+}
+
+func installForeignFence(t *testing.T, svc *activator.Service, h string, gen uint64) {
+ t.Helper()
+ ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
+ defer cancel()
+ if _, err := svc.FenceAndStop(ctx, h, gen); err != nil && !errors.Is(err, activator.ErrStopUnproven) {
+ t.Fatalf("installing the foreign fence: %v", err)
+ }
+}
+
+// A journal on a host another operation holds fenced is completed under
+// that operation's fence: the pass installs and lifts nothing. When the
+// completed endpoint is the fleet's intent there is nothing left to do,
+// and the journal goes.
+func TestRecoverUnderAForeignFenceCompletesAndRemovesAtTheIntent(t *testing.T) {
+ p, _ := box(t)
+ svc, _ := realService(t, p)
+ abortedNestedMove(t, p)
+ installForeignFence(t, svc, host, 5)
+ res := byHost(t, p.Recover(context.Background(), ModeNested))
+ for _, h := range []string{host, replica} {
+ if res[h].Outcome != "recovered" || JournalPresent(p.DataDir, h) || rowAt(t, p, h) != endpoints(t, p, h, ModeNested) {
+ t.Errorf("%s: %+v at %+v", h, res[h], rowAt(t, p, h))
+ }
+ }
+ if svc.Fenced(replica) {
+ t.Error("the unfenced host was left fenced")
+ }
+ if !svc.LiftFence(host, 5) {
+ t.Fatal("the foreign fence is not in place at its own generation")
+ }
+}
+
+// When the completed endpoint is not the intent, the pass cannot converge
+// the host without a fence of its own. It keeps the journal at done, which
+// keeps the host fenced if the other operation lifts its fence, and the
+// next recovery after that lift converges it.
+func TestRecoverUnderAForeignFenceKeepsTheJournalWhenNotAtTheIntent(t *testing.T) {
+ p, _ := box(t)
+ svc, _ := realService(t, p)
+ abortedNestedMove(t, p)
+ installForeignFence(t, svc, host, 5)
+ res := byHost(t, p.Recover(context.Background(), ModeFlat))
+ if r := res[host]; r.Outcome != "deferred" || !strings.Contains(r.Reason, "fence") {
+ t.Fatalf("held host: %+v", r)
+ }
+ if j, found, err := ReadJournal(p.DataDir, host); err != nil || !found || j.Phase != PhaseDone {
+ t.Fatalf("journal: %+v %v %v", j, found, err)
+ }
+ if rowAt(t, p, host) != endpoints(t, p, host, ModeNested) {
+ t.Errorf("row not completed to the journal's endpoint: %+v", rowAt(t, p, host))
+ }
+ if res[replica].Outcome != "recovered" {
+ t.Errorf("unfenced host: %+v", res[replica])
+ }
+ assertFlatUntouched(t, p, replica)
+ if !svc.Fenced(host) {
+ t.Fatal("the foreign fence was lifted")
+ }
+ if !svc.LiftFence(host, 5) {
+ t.Fatal("the foreign fence is not at its own generation")
+ }
+ // The other operation is gone: the next recovery converges the host.
+ if r := byHost(t, p.Recover(context.Background(), ModeFlat))[host]; r.Outcome != "recovered" {
+ t.Fatalf("after the foreign fence lifted: %+v", r)
+ }
+ assertFlatUntouched(t, p, host)
+ if svc.Fenced(host) {
+ t.Error("left fenced")
+ }
+}
+
+// Under a foreign fence the pass has no stop proof of its own, so it acts
+// only on a host with no live entry. Otherwise nothing changes.
+func TestRecoverUnderAForeignFenceDefersAnAwakeHost(t *testing.T) {
+ p, f := box(t)
+ abortedNestedMove(t, p)
+ f.fences = map[string]uint64{host: 5}
+ f.awake = map[string]bool{host: true}
+ f.events = nil
+ r := byHost(t, p.Recover(context.Background(), ModeNested))[host]
+ if r.Outcome != "deferred" {
+ t.Fatalf("result = %+v", r)
+ }
+ if j, found, _ := ReadJournal(p.DataDir, host); !found || j.Phase != PhaseFiles {
+ t.Fatalf("journal changed: %+v", j)
+ }
+ if rowAt(t, p, host) != endpoints(t, p, host, ModeFlat) {
+ t.Errorf("row changed: %+v", rowAt(t, p, host))
+ }
+ if f.fences[host] != 5 {
+ t.Errorf("foreign fence = %d", f.fences[host])
+ }
+ for _, e := range f.events {
+ if e == "lift "+host || e == "fence "+host {
+ t.Errorf("the pass touched the host's fence: %v", f.events)
+ }
+ }
+}
+
+// Completing a journal is never gated, but taking the host on toward a
+// nested intent is, as Converge(ModeNested) is. With the probe failing, a
+// revert completes and the host is released flat, which is safe.
+func TestRecoverGatesTheLegTowardNestedAndReleasesFlat(t *testing.T) {
+ p, f := box(t)
+ if res := byHost(t, p.Converge(context.Background(), ModeNested)); res[host].Outcome != "migrated" {
+ t.Fatalf("setup: %+v", res)
+ }
+ p.abortAt = "after-files"
+ p.Converge(context.Background(), ModeFlat)
+ p.abortAt = ""
+ calls := 0
+ p.NestedOK = func() error { calls++; return errors.New("probe: not ready") }
+ f.events = nil
+ res := byHost(t, p.Recover(context.Background(), ModeNested))
+ for _, h := range []string{host, replica} {
+ if r := res[h]; r.Outcome != "recovered" || !strings.Contains(r.Reason, "isolation not ready: probe: not ready") {
+ t.Errorf("%s: %+v", h, r)
+ }
+ assertFlatUntouched(t, p, h)
+ }
+ if calls != 1 {
+ t.Errorf("NestedOK called %d times, want once per recovery", calls)
+ }
+ want := []string{"fence " + host, "reload", "lift " + host, "fence " + replica, "reload", "lift " + replica}
+ if !reflect.DeepEqual(f.events, want) {
+ t.Fatalf("events = %v\nwant %v", f.events, want)
+ }
+}