amadan

carlos / platform Public

layout: hold a recovered host until it is at the intent, and check recorded absence

Recovery keeps the completed journal at done through the converge leg and
removes it only inside the new move's serialized section, right before
the new journal's exclusive write; a leg refused for any reason but the
isolation gate on a flat endpoint leaves the host held, not released.
Recovery refuses an artifact the journal records absent that exists at
either end. Converge completes journals before asking the isolation
gate, asks it at most once per pass, and refuses a host with no local
database before fencing it.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 03b2ea6e2c84c8128960a1f7e6fa15bccdf245e3 parent ff309e9
2 files changed, +366 −63
  • internal/layout/pass.go +151 −63
  • internal/layout/pass_linux_test.go +215 −0
diff --git a/internal/layout/pass.go b/internal/layout/pass.go
index 3d9eaed8c..b12e49087 100644
--- a/internal/layout/pass.go
+++ b/internal/layout/pass.go
@@ -152,16 +152,22 @@ func ready(rt registry.Route) bool { return rt.Status == "" || rt.Status == "rea
// in mode (design §8.2 steps 1 to 9). Rows are visited in host order so a
// log reads the same way twice. A host Recover reported is not visited
// again: recovery already converged it, or refused it.
+//
+// Recovery runs first and ungated, so a failing isolation probe never
+// leaves a crashed move fenced; its legs toward nested ask the gate. The
+// fresh moves to nested wait for the gate as a whole. The gate is asked at
+// most once per pass, and both share the answer.
func (p *Pass) Converge(ctx context.Context, mode Mode) []Result {
if mode != ModeNested && mode != ModeFlat {
return []Result{{Outcome: "refused", Reason: fmt.Sprintf("layout mode %q is not nested or flat", mode)}}
}
- if mode == ModeNested && p.NestedOK != nil {
- if err := p.NestedOK(); err != nil {
- return []Result{{Outcome: "refused", Reason: "isolation not ready: " + err.Error()}}
+ gate := p.nestedGate()
+ out := p.recover(ctx, mode, gate)
+ if mode == ModeNested {
+ if err := gate(); err != nil {
+ return append(out, Result{Outcome: "refused", Reason: "isolation not ready: " + err.Error()})
}
}
- out := p.Recover(ctx, mode)
seen := map[string]bool{}
for _, r := range out {
seen[r.Host] = true
@@ -217,12 +223,22 @@ func (p *Pass) migrateHost(ctx context.Context, host string, routes []registry.R
if err := Preflight(plan, routes); err != nil {
return Result{host, "refused", err.Error()}
}
+ // A host whose database is not on this box cannot move. Finding that
+ // out must not cost it a stop, and the pass directories and a
+ // destination check, on every pass. The same check under the fence
+ // stays the authority.
+ if db := plan.Artifacts[0]; db.Required {
+ if ok, err := lstatPresent(db.Src); err != nil || !ok {
+ return Result{host, "refused", fmt.Sprintf("db is missing at %s", db.Src)}
+ }
+ }
gen := p.generation()
if r, fence := p.quiesce(ctx, host, gen); fence != fenceOurs {
return r
}
defer p.release(host, gen)
- return p.shift(host, mode)
+ r, _ := p.shift(host, mode, nil)
+ return r
}
// fenceState is who holds a host's fence once quiesce returns.
@@ -289,51 +305,63 @@ func (p *Pass) release(host string, gen uint64) {
// classified can be stale: a move cut can have started, or the row been
// unflagged, re-added or migrated. Every return before the journal is
// written leaves nothing changed.
-func (p *Pass) shift(host string, mode Mode) Result {
+//
+// prior is a completed journal recovery is still holding the host with,
+// or nil. That journal is this pass's own, so it does not count as the
+// host being fenced here, and it is removed only immediately before the
+// new journal's exclusive write: until then it keeps the host fenced, so
+// a refusal anywhere in between leaves the host held, not released.
+// replaced reports whether prior was removed, and so whether the host's
+// fence is now the new journal (or nothing, after a completed move).
+func (p *Pass) shift(host string, mode Mode, prior *Journal) (res Result, replaced bool) {
fresh, err := p.Registry.Get(host)
if err != nil {
- return Result{host, "refused", "row: " + err.Error()}
+ return Result{host, "refused", "row: " + err.Error()}, false
}
if fresh == nil {
- return Result{host, "refused", "row vanished under the fence"}
+ return Result{host, "refused", "row vanished under the fence"}, false
}
if !ready(*fresh) {
- return Result{host, "refused", "status " + fresh.Status + " under the fence"}
+ return Result{host, "refused", "status " + fresh.Status + " under the fence"}, false
}
if !activator.ActivatorManaged(*fresh) {
- return Result{host, "refused", "no longer activator-managed under the fence"}
+ return Result{host, "refused", "no longer activator-managed under the fence"}, false
}
routes, err := p.Registry.List()
if err != nil {
- return Result{host, "refused", "list routes: " + err.Error()}
+ return Result{host, "refused", "list routes: " + err.Error()}, false
}
want, from := activator.LayoutNested, activator.LayoutFlat
if mode == ModeFlat {
want, from = activator.LayoutFlat, activator.LayoutNested
}
- switch layout, reason := p.Classify(*fresh, routes); layout {
+ in := p.layoutInputs(routes)
+ if prior != nil {
+ in.Journal = func(h string) bool { return h != host && JournalPresent(p.DataDir, h) }
+ }
+ switch layout, reason := activator.Classify(*fresh, in); layout {
case want:
- return Result{host, "unchanged", ""}
+ return Result{host, "unchanged", ""}, false
case from:
default:
- return Result{host, "refused", fmt.Sprintf("%s under the fence: %s", layout, reason)}
+ return Result{host, "refused", fmt.Sprintf("%s under the fence: %s", layout, reason)}, false
}
plan, err := PlanFor(p.DataDir, p.socketDir(), host, mode)
if err != nil {
- return Result{host, "refused", err.Error()}
+ return Result{host, "refused", err.Error()}, false
}
if err := Preflight(plan, routes); err != nil {
- return Result{host, "refused", err.Error()}
+ return Result{host, "refused", err.Error()}, false
}
// Both directions: a revert renames out of the raw directory, so it
// must be a real directory too, and it is cheaper to find a planted
// link here than after the journal has fenced the host.
if err := EnsureDirs(p.DataDir, p.socketDir(), host, p.SocketsGID); err != nil {
- return Result{host, "refused", err.Error()}
+ return Result{host, "refused", err.Error()}, false
}
aside, err := DestinationCheck(plan, p.now)
if err != nil {
- return Result{host, "refused", err.Error()}
+ return Result{host, "refused", err.Error()}, false
}
present := make([]bool, len(plan.Artifacts))
for i, a := range plan.Artifacts {
@@ -344,31 +372,52 @@ func (p *Pass) shift(host string, mode Mode) Result {
}
ok, err := lstatPresent(a.Src)
if err != nil {
- return Result{host, "refused", err.Error()}
+ return Result{host, "refused", err.Error()}, false
}
if a.Required && !ok {
- return Result{host, "refused", fmt.Sprintf("%s is missing at %s", a.Name, a.Src)}
+ return Result{host, "refused", fmt.Sprintf("%s is missing at %s", a.Name, a.Src)}, false
}
present[i] = ok
plan.Artifacts[i].Present = ok
}
- var res Result
- p.serialize(func() { res = p.commit(plan, plan.Journal(present), aside) })
+ p.serialize(func() { res, replaced = p.commit(plan, plan.Journal(present), aside, prior) })
if res.Outcome == "migrated" || res.Outcome == "reverted" {
p.reload()
}
- return res
+ return res, replaced
}
// commit is steps 6 to 9 up to the journal's removal. The caller reloads
// routing afterwards, outside Serialize's lock and still under the fence.
// Every return after the journal is written leaves it for Recover, and the
// journal keeps the host fenced until then.
-func (p *Pass) commit(plan *Plan, j *Journal, aside []string) Result {
+func (p *Pass) commit(plan *Plan, j *Journal, aside []string, prior *Journal) (Result, bool) {
host := plan.Host
+ replaced := false
+ if prior != nil {
+ if err := RemoveJournal(p.DataDir, host); err != nil {
+ return Result{host, "refused", err.Error()}, false
+ }
+ replaced = true
+ }
if err := WriteJournal(p.DataDir, j); err != nil {
- return Result{host, "refused", err.Error()}
+ // Put the held journal back: without it, or the new one, nothing
+ // would keep the host from being released at an endpoint that is
+ // not the intent.
+ if replaced {
+ if rerr := WriteJournal(p.DataDir, prior); rerr != nil {
+ return Result{host, "refused", fmt.Sprintf("%v; restoring the held journal: %v", err, rerr)}, true
+ }
+ }
+ return Result{host, "refused", err.Error()}, false
}
+ r := p.commitWritten(plan, j, aside)
+ return r, replaced
+}
+
+// commitWritten is commit after the journal is on disk.
+func (p *Pass) commitWritten(plan *Plan, j *Journal, aside []string) Result {
+ host := plan.Host
if p.abortAt == "after-journal" {
return Result{host, "refused", "aborted"}
}
@@ -452,9 +501,14 @@ func (p *Pass) moveFiles(plan *Plan, j *Journal) error {
// §8.2, "Recovery is the journal"). It runs at agent start and at the top
// of every Converge. intent is the fleet's recorded layout: "" when none
// was ever recorded, IntentUnreadable when the agent could not read it.
-// Recovery is never gated on the isolation probe: it is how a host gets
-// out of the fenced state.
+// Completing a journal is never gated on the isolation probe, since that
+// is how a host gets out of the fenced state; a leg after it toward a
+// nested intent is (see recoverHost).
func (p *Pass) Recover(ctx context.Context, intent Mode) []Result {
+ return p.recover(ctx, intent, p.nestedGate())
+}
+
+func (p *Pass) recover(ctx context.Context, intent Mode, gate func() error) []Result {
entries, err := os.ReadDir(journalDir(p.DataDir))
if errors.Is(err, fs.ErrNotExist) {
return nil
@@ -483,7 +537,6 @@ func (p *Pass) Recover(ctx context.Context, intent Mode) []Result {
byHost[rt.Host] = rt
}
var out []Result
- gate := p.nestedGate()
for _, host := range names {
if ctx.Err() != nil {
break
@@ -498,7 +551,7 @@ func (p *Pass) Recover(ctx context.Context, intent Mode) []Result {
}
// nestedGate asks NestedOK at most once, on first use, and remembers the
-// answer for the rest of one recovery, as Converge asks it once per pass.
+// answer for the rest of one pass.
func (p *Pass) nestedGate() func() error {
asked := false
var answer error
@@ -524,14 +577,18 @@ func notActing(err error) string {
// recoverHost checks one journal against the row and both derived
// endpoints, completes it to its own end under a fence, and then, still
-// under that fence, converges the host to intent. Releasing the host
-// between the two would let a forged or stale journal run it for a moment
-// in a layout the fleet did not ask for (design §3.3).
+// under that fence, converges the host to intent. The host is released
+// only once it stands at the intent (design §3.3): while the converge leg
+// runs, the completed journal stays on disk at done and keeps the host
+// fenced, and if the leg refuses, for any reason, it stays there. A flat
+// tenant can forge a valid revert journal for a nested sibling; completing
+// it costs that sibling a round trip, and if the way back is refused the
+// sibling waits fenced for an operator rather than running flat.
//
// Completing a journal is never gated on the isolation probe; the leg
// after it toward a nested intent is, exactly as Converge(ModeNested) is.
-// A host the gate stops there is released at its completed endpoint, which
-// is then flat and safe to serve.
+// That is the one refused leg that releases the host: its completed
+// endpoint is then flat, which is safe to serve.
//
// When another operation's fence already holds the host (the residence
// move's cut), the pass completes the journal under that fence, installing
@@ -591,50 +648,83 @@ func (p *Pass) recoverHost(ctx context.Context, host string, byHost map[string]r
if err != nil {
return Result{host, "refused", notActing(err)}
}
+ converge := fence == fenceOurs && (intent == ModeNested || intent == ModeFlat) && intent != j.Mode
var holdReason string
switch {
case fence == fenceHeld && intent != j.Mode:
holdReason = fmt.Sprintf("completed to %s under another operation's fence; journal kept at done until that fence is lifted", j.Mode)
case intent == IntentUnreadable && j.Mode == ModeFlat:
- // A flat tenant can forge a valid revert journal for a nested
- // host, and releasing it flat would run it unisolated until an
- // intent is read.
+ // Releasing a completed revert flat before the intent is known is
+ // the forged-revert window again.
holdReason = "completed to flat; journal kept at done until the layout intent can be read"
}
+ remove := holdReason == "" && !converge
var res Result
- p.serialize(func() { res = p.complete(plan, j, atFrom, holdReason) })
- if res.Outcome == "recovered" || res.Outcome == "deferred" {
- // Both flipped the row (or found it flipped); a held host is
- // still fenced by its journal, but routing should say where it is.
- p.reload()
+ p.serialize(func() { res = p.complete(plan, j, atFrom, remove) })
+ if res.Outcome != "recovered" {
+ return res
}
- if res.Outcome != "recovered" || fence == fenceHeld || intent == "" || intent == IntentUnreadable || intent == j.Mode {
+ // The row flipped (or was found flipped). A host still held by its
+ // journal is refused wakes by that, but routing should say where it is.
+ p.reload()
+ if holdReason != "" {
+ p.logf("layout: %s: held -> %s", host, holdReason)
+ return Result{host, "deferred", holdReason}
+ }
+ if !converge {
return res
}
if intent == ModeNested {
- if err := gate(); err != nil {
- p.logf("layout: %s: released -> %s; not converging to nested: isolation not ready: %v", host, j.Mode, err)
- return Result{host, "recovered", fmt.Sprintf("completed to %s; not converging to nested: isolation not ready: %v", j.Mode, err)}
+ if gerr := gate(); gerr != nil {
+ var rerr error
+ p.serialize(func() { rerr = RemoveJournal(p.DataDir, host) })
+ if rerr != nil {
+ return Result{host, "refused", rerr.Error()}
+ }
+ p.logf("layout: %s: released -> %s; not converging to nested: isolation not ready: %v", host, j.Mode, gerr)
+ return Result{host, "recovered", fmt.Sprintf("completed to %s; not converging to nested: isolation not ready: %v", j.Mode, gerr)}
}
}
- next := p.shift(host, intent)
- switch next.Outcome {
- case "migrated", "reverted":
+ next, replaced := p.shift(host, intent, j)
+ switch {
+ case next.Outcome == "migrated" || next.Outcome == "reverted":
return Result{host, "recovered", fmt.Sprintf("completed to %s, then converged to %s", j.Mode, intent)}
- case "unchanged":
- return res
+ case !replaced:
+ reason := fmt.Sprintf("completed to %s; journal kept at done: converging to %s: %s %s", j.Mode, intent, next.Outcome, next.Reason)
+ p.logf("layout: %s: held -> %s", host, reason)
+ return Result{host, "deferred", reason}
}
return Result{host, next.Outcome, fmt.Sprintf("completed to %s; converging to %s: %s", j.Mode, intent, next.Reason)}
}
// complete runs the journal's remaining phases (design §8.2's recovery
-// list). A done journal whose row is still at from is refused rather than
-// removed: a crash cannot produce it, since done is written only after the
-// row's swap committed, and removing it would release a row that names the
-// files' old place. A non-empty holdReason completes everything but the
-// journal's removal and reports the host deferred with that reason.
-func (p *Pass) complete(plan *Plan, j *Journal, atFrom bool, holdReason string) Result {
+// list), and removes the journal only when remove is set; otherwise it
+// stops at done and the journal keeps holding the host.
+//
+// Two states a crash cannot produce are refused rather than acted on. An
+// artifact the journal records absent that exists at either end: the host
+// has been fenced since the plan was made, so nothing recorded absent can
+// have appeared, and moving the rest without it would leave it behind (a
+// forged journal can use exactly that to strand a database file). And a
+// done journal whose row is still at from: done is written only after the
+// row's swap committed, and releasing it would leave a row that names the
+// files' old place.
+func (p *Pass) complete(plan *Plan, j *Journal, atFrom, remove bool) Result {
host, phase := plan.Host, j.Phase
+ for i, a := range plan.Artifacts {
+ if a.Name == "socket" || j.Artifacts[i].Present {
+ continue
+ }
+ for _, path := range []string{a.Src, a.Dst} {
+ present, err := lstatPresent(path)
+ if err != nil {
+ return Result{host, "refused", notActing(err)}
+ }
+ if present {
+ return Result{host, "refused", fmt.Sprintf("journal records %s absent, but it exists at %s; not acting", a.Name, path)}
+ }
+ }
+ }
switch j.Phase {
case PhaseFiles:
if err := p.moveFiles(plan, j); err != nil {
@@ -660,12 +750,10 @@ func (p *Pass) complete(plan *Plan, j *Journal, atFrom bool, holdReason string)
default:
return Result{host, "refused", fmt.Sprintf("journal phase %q unknown; not acting", j.Phase)}
}
- if holdReason != "" {
- p.logf("layout: %s: held -> %s", host, holdReason)
- return Result{host, "deferred", holdReason}
- }
- if err := RemoveJournal(p.DataDir, host); err != nil {
- return Result{host, "refused", err.Error()}
+ if remove {
+ if err := RemoveJournal(p.DataDir, host); err != nil {
+ return Result{host, "refused", err.Error()}
+ }
}
p.logf("layout: %s: recovered -> %s (journal phase was %s)", host, j.Mode, phase)
return Result{host, "recovered", string(j.Mode)}
diff --git a/internal/layout/pass_linux_test.go b/internal/layout/pass_linux_test.go
index 812df8981..908b90735 100644
--- a/internal/layout/pass_linux_test.go
+++ b/internal/layout/pass_linux_test.go
@@ -903,3 +903,218 @@ func TestRecoverGatesTheLegTowardNestedAndReleasesFlat(t *testing.T) {
t.Fatalf("events = %v\nwant %v", f.events, want)
}
}
+
+// addFlatRow adds a fresh flat managed row with its files, after any setup
+// that would otherwise have moved it.
+func addFlatRow(t *testing.T, p *Pass, h string) {
+ t.Helper()
+ if err := p.Registry.Upsert(registry.Route{Host: h, Kind: "instance", Hibernate: true,
+ Socket: filepath.Join(p.SocketDir, h+".sock"), DBPath: provision.DefaultDBPath(p.DataDir, h)}); err != nil {
+ t.Fatal(err)
+ }
+ seedFlat(t, p.DataDir, h)
+}
+
+// forgeFlatJournal writes a well-formed revert journal for a nested host,
+// recording each artifact present exactly when it is at its nested source,
+// except the names in absent, which it records absent regardless. It is
+// what a flat tenant sharing the agent's uid can write into <data>/layout.
+func forgeFlatJournal(t *testing.T, p *Pass, h string, absent ...string) {
+ t.Helper()
+ plan, err := PlanFor(p.DataDir, p.SocketDir, h, ModeFlat)
+ if err != nil {
+ t.Fatal(err)
+ }
+ present := make([]bool, len(plan.Artifacts))
+ for i, a := range plan.Artifacts {
+ ok, _ := lstatPresent(a.Src)
+ present[i] = ok && a.Name != "socket" && !slicesContains(absent, a.Name)
+ }
+ if err := WriteJournal(p.DataDir, plan.Journal(present)); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func slicesContains(xs []string, x string) bool {
+ for _, v := range xs {
+ if v == x {
+ return true
+ }
+ }
+ return false
+}
+
+// A journal completed to an endpoint that is not the fleet's intent keeps
+// the host fenced until the converge leg succeeds. If the leg refuses, the
+// journal stays at done: a forged revert must not leave a nested host
+// released flat.
+func TestRecoverHoldsTheJournalWhenTheLegTowardTheIntentRefuses(t *testing.T) {
+ p, f := box(t)
+ if res := byHost(t, p.Converge(context.Background(), ModeNested)); res[host].Outcome != "migrated" {
+ t.Fatalf("setup: %+v", res)
+ }
+ forgeFlatJournal(t, p, host)
+ // A route that owns <host>.config.env under the flat rules makes the
+ // way back to nested refuse at the ownership preflight.
+ if err := p.Registry.Upsert(registry.Route{Host: host + ".config", Kind: "instance",
+ Socket: filepath.Join(p.SocketDir, host+".config.sock"), DBPath: provision.DefaultDBPath(p.DataDir, host+".config")}); err != nil {
+ t.Fatal(err)
+ }
+ r := byHost(t, p.Recover(context.Background(), ModeNested))[host]
+ if r.Outcome != "deferred" || !strings.Contains(r.Reason, "config") {
+ t.Fatalf("result = %+v", r)
+ }
+ j, found, err := ReadJournal(p.DataDir, host)
+ if err != nil || !found || j.Phase != PhaseDone || j.Mode != ModeFlat {
+ t.Fatalf("journal not held at done: %+v %v %v", j, found, err)
+ }
+ if len(f.fences) != 0 {
+ t.Errorf("fences: %v", f.fences)
+ }
+ // Every later pass holds it the same way; it is never released flat.
+ if r := byHost(t, p.Converge(context.Background(), ModeNested))[host]; r.Outcome != "deferred" || !JournalPresent(p.DataDir, host) {
+ t.Fatalf("second pass: %+v", r)
+ }
+}
+
+// The attack the hold closes, end to end: a forged revert that records a
+// real nested artifact absent. Recovery refuses it outright, and the host
+// stays nested and fenced.
+func TestRecoverDoesNotReleaseANestedHostFlatOverAForgedRevert(t *testing.T) {
+ p, _ := box(t)
+ if res := byHost(t, p.Converge(context.Background(), ModeNested)); res[host].Outcome != "migrated" {
+ t.Fatalf("setup: %+v", res)
+ }
+ shadow := filepath.Join(rawInstanceDir(p.DataDir, host), shadowName(host))
+ if _, err := os.Stat(shadow); err != nil {
+ t.Fatalf("setup: %v", err)
+ }
+ forgeFlatJournal(t, p, host, "litestream-shadow")
+ r := byHost(t, p.Recover(context.Background(), ModeNested))[host]
+ if r.Outcome == "recovered" {
+ t.Fatalf("result = %+v", r)
+ }
+ if rowAt(t, p, host) != endpoints(t, p, host, ModeNested) {
+ t.Errorf("row released flat: %+v", rowAt(t, p, host))
+ }
+ if !JournalPresent(p.DataDir, host) {
+ t.Error("journal removed; the host is no longer fenced")
+ }
+}
+
+// A journal's "absent" is a claim a crash cannot contradict: the host has
+// been fenced since the plan was made, so nothing recorded absent can have
+// appeared at either end. Recovery refuses one that has, naming it.
+func TestRecoverRefusesAnArtifactRecordedAbsentThatExists(t *testing.T) {
+ for _, end := range []string{"source", "destination"} {
+ t.Run(end, func(t *testing.T) {
+ p, _ := box(t)
+ abortedNestedMove(t, p)
+ plan, _ := PlanFor(p.DataDir, p.SocketDir, host, ModeNested)
+ sidecar := artifactNamed(t, plan, "sidecar-env")
+ path := sidecar.Src
+ if end == "destination" {
+ path = sidecar.Dst
+ }
+ if err := os.WriteFile(path, []byte("X=1\n"), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ r := byHost(t, p.Recover(context.Background(), ModeNested))[host]
+ if r.Outcome != "refused" || !strings.Contains(r.Reason, path) || !strings.Contains(r.Reason, "absent") {
+ t.Fatalf("result = %+v", r)
+ }
+ if j, found, _ := ReadJournal(p.DataDir, host); !found || j.Phase != PhaseFiles {
+ t.Fatalf("journal changed: %+v", j)
+ }
+ })
+ }
+}
+
+// Completing a journal is never gated, so a failing probe does not leave a
+// crashed move fenced. Only fresh moves wait for the probe.
+func TestConvergeNestedCompletesAJournalWhileTheProbeFails(t *testing.T) {
+ p, _ := box(t)
+ abortedNestedMove(t, p)
+ addFlatRow(t, p, "f.example.com")
+ calls := 0
+ p.NestedOK = func() error { calls++; return errors.New("probe: not ready") }
+ res := byHost(t, p.Converge(context.Background(), ModeNested))
+ for _, h := range []string{host, replica} {
+ if res[h].Outcome != "recovered" || JournalPresent(p.DataDir, h) || rowAt(t, p, h) != endpoints(t, p, h, ModeNested) {
+ t.Errorf("%s: %+v", h, res[h])
+ }
+ }
+ if r := res[""]; r.Outcome != "refused" || r.Reason != "isolation not ready: probe: not ready" {
+ t.Errorf("pass result: %+v", r)
+ }
+ if _, listed := res["f.example.com"]; listed {
+ t.Errorf("a fresh move ran under a failing probe: %+v", res["f.example.com"])
+ }
+ assertFlatUntouched(t, p, "f.example.com")
+ if calls != 1 {
+ t.Errorf("NestedOK called %d times", calls)
+ }
+}
+
+// One pass asks the probe once, whether the question comes from a
+// recovery's leg toward nested or from the fresh moves.
+func TestConvergeAsksTheIsolationGateOncePerPass(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ gate error
+ }{{"failing", errors.New("probe: not ready")}, {"ready", nil}} {
+ t.Run(tc.name, func(t *testing.T) {
+ p, _ := box(t)
+ if res := byHost(t, p.Converge(context.Background(), ModeNested)); res[host].Outcome != "migrated" {
+ t.Fatalf("setup: %+v", res)
+ }
+ p.abortAt = "after-files"
+ p.Converge(context.Background(), ModeFlat)
+ p.abortAt = ""
+ addFlatRow(t, p, "f.example.com")
+ calls := 0
+ p.NestedOK = func() error { calls++; return tc.gate }
+ res := byHost(t, p.Converge(context.Background(), ModeNested))
+ if calls != 1 {
+ t.Fatalf("NestedOK called %d times in one pass", calls)
+ }
+ for _, h := range []string{host, replica} {
+ if res[h].Outcome != "recovered" {
+ t.Errorf("%s: %+v", h, res[h])
+ }
+ }
+ if tc.gate != nil {
+ assertFlatUntouched(t, p, host)
+ assertFlatUntouched(t, p, "f.example.com")
+ } else if res["f.example.com"].Outcome != "migrated" || rowAt(t, p, host) != endpoints(t, p, host, ModeNested) {
+ t.Errorf("ready probe: %+v", res)
+ }
+ })
+ }
+}
+
+// A host whose database is not on this box cannot move; finding that out
+// must not cost it a stop, or directories and a destination check, on
+// every pass.
+func TestConvergeRefusesAHostWithNoLocalDatabaseWithoutFencingIt(t *testing.T) {
+ p, f := box(t)
+ db := provision.DefaultDBPath(p.DataDir, host)
+ if err := os.Remove(db); err != nil {
+ t.Fatal(err)
+ }
+ res := byHost(t, p.Converge(context.Background(), ModeNested))
+ if r := res[host]; r.Outcome != "refused" || !strings.Contains(r.Reason, "db is missing at "+db) {
+ t.Fatalf("result = %+v", r)
+ }
+ for _, e := range f.events {
+ if strings.HasSuffix(e, " "+host) {
+ t.Errorf("the host's fence was touched: %v", f.events)
+ }
+ }
+ if _, err := os.Lstat(rawInstanceDir(p.DataDir, host)); !errors.Is(err, os.ErrNotExist) {
+ t.Errorf("directories were made for a host that cannot move: %v", err)
+ }
+ if res[replica].Outcome != "migrated" {
+ t.Errorf("the other host: %+v", res[replica])
+ }
+}