amadan

carlos / platform Public

region ha: a source whose reconcile is delayed past the completion holds its row for the completed op until its park-local retires it (L18 finding 3).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude a092ddc23c8ca1c085f13264bdbc4639118a5a37 parent 987014e
5 files changed, +493 −22
  • internal/host/relocationsourcehold.go +7 −4
  • internal/host/relocationsourcehold_test.go +155 −10
  • internal/hostbox/reconcile.go +78 −7
  • internal/hostbox/reconcile_relocation_catchup_test.go +251 −0
  • internal/hostbox/relocationsource.go +2 −1
diff --git a/internal/host/relocationsourcehold.go b/internal/host/relocationsourcehold.go
index fedbbf187..2e3d01855 100644
--- a/internal/host/relocationsourcehold.go
+++ b/internal/host/relocationsourcehold.go
@@ -12,10 +12,13 @@ import (
// (hostbox.AdmissionFenceClosed), and HoldRow holds the row for the op in
// "relocating" (hostbox.RegistryHoldRow). The source in releasing is then
// held at its fence's close, and every other relocating participant keeps
-// an existing row held. Nothing here releases a source's hold: the
-// fenced installation never serves again, so the completion queues the
-// source's park-local (hostclaim.CompleteRelocation), whose retirement
-// deletes the row by ownership, held or not. With the flag off both hooks
+// an existing row held. A source whose reconcile is delayed past the
+// completion is held then, for the completed op, by reconcile's catch-up
+// hold while it owes that op's park-local (L18 joint round, finding 3).
+// Nothing here releases a source's hold: the fenced installation never
+// serves again, so the completion queues the source's park-local
+// (hostclaim.CompleteRelocation), whose retirement deletes the row by
+// ownership, held or not. With the flag off both hooks
// stay nil and reconcile is main's: Serving alone, which refuses every
// relocating claim with every row unchanged.
func relocationSourceRows(getenv func(string) string, r *hostbox.Reconciler, guard hostguard.Guard, reg *registry.Registry) {
diff --git a/internal/host/relocationsourcehold_test.go b/internal/host/relocationsourcehold_test.go
index 92abfcc54..030e67331 100644
--- a/internal/host/relocationsourcehold_test.go
+++ b/internal/host/relocationsourcehold_test.go
@@ -194,6 +194,16 @@ func regionHA(on bool) func(string) string {
// session, then the real serving CAS (hostclaim.CompleteRelocation),
// which queues the source's park-local unless a fence discharged it.
func (b *sourceHoldBox) complete() hostclaim.Observed {
+ b.t.Helper()
+ obs := b.completeOwing()
+ if !hostclaim.OwesLocal(obs.Claim, "box") {
+ b.t.Fatalf("after the completion: obligations %+v", obs.Claim.Obligations)
+ }
+ return obs
+}
+
+// completeOwing is complete without asking which boxes owe a park-local.
+func (b *sourceHoldBox) completeOwing() hostclaim.Observed {
b.t.Helper()
b.update(func(c *hostclaim.Claim) {
h := c.Holder
@@ -214,7 +224,7 @@ func (b *sourceHoldBox) complete() hostclaim.Observed {
if err != nil {
b.t.Fatalf("the serving CAS: %v", err)
}
- if obs.Claim.Op.Kind != hostclaim.KindServing || obs.Claim.Op.Exec != "dst" || !hostclaim.OwesLocal(obs.Claim, "box") {
+ if obs.Claim.Op.Kind != hostclaim.KindServing || obs.Claim.Op.Exec != "dst" {
b.t.Fatalf("after the completion: op %+v obligations %+v", obs.Claim.Op, obs.Claim.Obligations)
}
return obs
@@ -236,12 +246,28 @@ func (b *sourceHoldBox) leaves(what string, before registry.Route, record []byte
}
}
+// replays asserts reconcile, dry run or not, answers the catch-up hold's
+// replay (L18 finding 3): nil, with the row exactly as before, the fence
+// not asked, nothing activated or written, and the guard record as it was.
+func (b *sourceHoldBox) replays(what string, before registry.Route, record []byte) {
+ b.t.Helper()
+ asked, writes := b.asked, b.writes
+ for _, dry := range []bool{true, false} {
+ if err := b.install(dry); err != nil {
+ b.t.Fatalf("%s (dry run %v): %v", what, dry, err)
+ }
+ }
+ if got := b.row(); got == nil || *got != before || b.asked != asked || b.writes != writes || !bytes.Equal(b.guardRecord(), record) {
+ b.t.Fatalf("%s: row %+v (was %+v) fence asked %d writes %d", what, got, before, b.asked-asked, b.writes-writes)
+ }
+}
+
// Task 10b test 3. The completion: the source's row, held at its fence's
-// close, stays held through the real serving CAS. Reconcile neither
-// renews nor releases it (the source is no longer a participant, and
-// reconcile under a serving claim is main's), and the completion queues
-// the source's park-local, whose retirement deletes the held row by its
-// ownership: that ends the hold, and nothing is minted after. A failover
+// close, stays held through the real serving CAS. Reconcile never
+// releases it: under the serving claim the source owes the completed op's
+// park-local, so reconcile's catch-up hold (L18 finding 3) replays the
+// hold and writes nothing. The park-local's retirement deletes the held
+// row by its ownership: that ends the hold, and nothing is minted after. A failover
// from a fenced source: the source is on lost, so reconcile neither asks
// its fence nor holds its row, whether the row was ready or already held
// when the box was fenced, both while the failover runs and after its
@@ -260,11 +286,10 @@ func TestRelocationSourceHoldEndsWithTheCompletionOrTheParkLocal(t *testing.T) {
t.Fatalf("held row %+v holds %d", held, b.holds)
}
b.complete()
- b.leaves("the source after the completion", held, b.guardRecord())
+ b.replays("the source after the completion", held, b.guardRecord())
b.retireRow()
- holds := b.holds
- if err := b.install(false); !errors.Is(err, hostbox.ErrChanged) || b.row() != nil || b.holds != holds {
- t.Fatalf("after the park-local: %v row %+v holds %d", err, b.row(), b.holds-holds)
+ if err := b.install(false); !errors.Is(err, hostbox.ErrChanged) || b.row() != nil || b.writes != 0 {
+ t.Fatalf("after the park-local: %v row %+v writes %d", err, b.row(), b.writes)
}
})
for _, fencedHeld := range []bool{false, true} {
@@ -355,3 +380,123 @@ func TestRelocationSourceHoldFollowsTheFlag(t *testing.T) {
})
}
}
+
+// L18 joint round, finding 3: the source releases and its fence closes,
+// its reconcile is delayed, the target completes, and only then does the
+// source reconcile. The claim serves on another box, so Task 10b's
+// relocating scope no longer reaches it; the catch-up hold does. Through
+// the production hooks (relocationSourceRows) and the real reconcile: the
+// row ends held for the completed op, never ready, its ownership and
+// every other column as they were, the fence not asked, nothing activated
+// and the claim and the guard record unchanged; a replay changes nothing;
+// the park-local's retirement then ends it and nothing is minted.
+//
+// The guards: a row another op holds is never taken; with region HA off
+// reconcile is main's (refused, the row ready and unchanged); a box that
+// was not the relocation's source, or owes a park-local for another op
+// only, does nothing; a box on lost is left to its park-local (Task 10b
+// decision 3, the failover test above); a completion record that cannot
+// be read is its error, never "no completion", with the row unchanged.
+func TestRelocationSourceCatchUpHoldAfterTheCompletion(t *testing.T) {
+ t.Run("delayed source reconcile", func(t *testing.T) {
+ b := newSourceHoldBox(t, regionHA(true))
+ minted := *b.row()
+ b.releasing()
+ b.closeFence()
+ done := b.complete()
+ record := b.guardRecord()
+ if err := b.install(false); err != nil {
+ t.Fatalf("the source's first reconcile after the completion: %v", err)
+ }
+ got := *b.row()
+ if got.Status != "relocating" || got.HoldOp != "rel-1" || b.holds != 1 || b.asked != 0 || b.writes != 0 {
+ t.Fatalf("the source's row %+v holds %d fence asked %d writes %d", got, b.holds, b.asked, b.writes)
+ }
+ held := got
+ held.Status, held.HoldOp = minted.Status, minted.HoldOp
+ if held != minted {
+ t.Fatalf("the hold changed more than status and hold_op: %+v (minted %+v)", got, minted)
+ }
+ if now, err := hostclaim.Read(t.Context(), b.store, sourceHoldHost); err != nil || now.ETag != done.ETag || !bytes.Equal(b.guardRecord(), record) {
+ t.Fatalf("the catch-up hold wrote the claim or the guard record: %v", err)
+ }
+ b.replays("the replay", got, record)
+ b.retireRow()
+ if err := b.install(false); !errors.Is(err, hostbox.ErrChanged) || b.row() != nil || b.writes != 0 {
+ t.Fatalf("after the park-local: %v row %+v writes %d", err, b.row(), b.writes)
+ }
+ })
+ t.Run("another op's row is never taken", func(t *testing.T) {
+ b := newSourceHoldBox(t, regionHA(true))
+ b.releasing()
+ b.closeFence()
+ if ok, err := b.reg.SetHoldIf(sourceHoldHost, "moving", "mv-9", ""); err != nil || !ok {
+ t.Fatalf("holding the row for another op: %v %v", ok, err)
+ }
+ before := *b.row()
+ b.complete()
+ for _, dry := range []bool{true, false} {
+ if err := b.install(dry); !dry && !errors.Is(err, hostbox.ErrChanged) {
+ t.Fatalf("another op's row: %v", err)
+ }
+ }
+ if got := b.row(); *got != before || b.writes != 0 {
+ t.Fatalf("another op's row was taken: %+v (was %+v) writes %d", got, before, b.writes)
+ }
+ })
+ t.Run("region HA off", func(t *testing.T) {
+ b := newSourceHoldBox(t, regionHA(false))
+ b.releasing()
+ b.closeFence()
+ b.complete()
+ b.leaves("region HA off after the completion", *b.row(), b.guardRecord())
+ if got := b.row(); got.Status != "ready" || got.HoldOp != "" {
+ t.Fatalf("region HA off: row %+v", got)
+ }
+ })
+ t.Run("not the source", func(t *testing.T) {
+ b := newSourceHoldBox(t, regionHA(true))
+ before, record := *b.row(), b.guardRecord()
+ // A drain of the host from "other" to "dst": this box holds a
+ // stale local record and row, but was never a participant.
+ b.update(func(c *hostclaim.Claim) {
+ c.Op = hostclaim.Op{Kind: hostclaim.KindRelocating, ID: "rel-1", Phase: hostclaim.PhaseReleasing, Exec: "other", Participants: []string{"other", "dst"},
+ Intent: map[string]string{"region": "ie", "reason": "drain", "source_box": "other", "target_box": "dst"}, Journal: map[string]string{"exec_ack.other": "{}", "exec_ack.dst": "{}"}}
+ })
+ done := b.completeOwing()
+ if hostclaim.OwesLocal(done.Claim, "box") || !hostclaim.OwesLocal(done.Claim, "other") {
+ t.Fatalf("the completion's park-locals: %+v", done.Claim.Obligations)
+ }
+ b.leaves("a box that was not the source", before, record)
+ })
+ t.Run("a park-local for another op only", func(t *testing.T) {
+ b := newSourceHoldBox(t, regionHA(true))
+ b.releasing()
+ b.closeFence()
+ b.complete()
+ b.update(func(c *hostclaim.Claim) {
+ for i := range c.Obligations {
+ if c.Obligations[i].Kind == "park-local" && c.Obligations[i].Box == "box" {
+ c.Obligations[i].Op = "rel-0"
+ }
+ }
+ })
+ b.leaves("a park-local for another op", *b.row(), b.guardRecord())
+ })
+ t.Run("an unreadable completion record", func(t *testing.T) {
+ b := newSourceHoldBox(t, regionHA(true))
+ b.releasing()
+ b.closeFence()
+ b.complete()
+ b.update(func(c *hostclaim.Claim) { c.Op.Journal[hostclaim.JournalCompleted] = "{" })
+ before, record := *b.row(), b.guardRecord()
+ for _, dry := range []bool{true, false} {
+ if err := b.install(dry); err == nil || errors.Is(err, hostbox.ErrChanged) || !errors.Is(err, hostclaim.ErrRelocation) {
+ t.Fatalf("an unreadable completion record (dry run %v): %v (want the read's error)", dry, err)
+ }
+ }
+ if got := b.row(); *got != before || b.holds != 0 || b.writes != 0 || !bytes.Equal(b.guardRecord(), record) {
+ t.Fatalf("an unreadable completion record changed something: row %+v holds %d writes %d", got, b.holds, b.writes)
+ }
+ })
+}
diff --git a/internal/hostbox/reconcile.go b/internal/hostbox/reconcile.go
index 7412a9d3e..e07700293 100644
--- a/internal/hostbox/reconcile.go
+++ b/internal/hostbox/reconcile.go
@@ -2,6 +2,7 @@ package hostbox
import (
"context"
+ "slices"
"time"
"github.com/carlosframework/platform/internal/hostclaim"
@@ -54,16 +55,57 @@ type Reconciler struct {
// relocationAware is region HA on: Task 10b sets FenceClosed and HoldRow
// only with CARLOS_REGION_HA (host's relocationSourceRows). With neither,
// reconcile is today's: the serving executor alone (Serving), and no
-// hold-only path.
+// hold-only path, the completed relocation's catch-up hold included.
func (r Reconciler) relocationAware() bool { return r.FenceClosed != nil || r.HoldRow != nil }
// relocates is the row decision's scope: a relocating claim with region HA
// on. Every other operation, moving included, keeps main's path (Serving),
-// HA on or off: moves owns its holds, in provision's pass.
+// HA on or off: moves owns its holds, in provision's pass. The one
+// exception is a serving claim's completed relocation, whose catch-up hold
+// (completedHold) is decided apart from this scope.
func (r Reconciler) relocates(c *hostclaim.Claim) bool {
return r.relocationAware() && c != nil && c.Op.Kind == hostclaim.KindRelocating
}
+// completedHold is the catch-up hold (L18 joint round, finding 3): the
+// source's fence closed in releasing, but its next reconcile ran only once
+// the target had completed, so the claim serves on another box and
+// rowDecision's scope (relocates) no longer reaches it, and main's Serving
+// would refuse with the row left ready, which the edge serves, until the
+// park-local retires it. Completion waits on no acknowledgement of the
+// source's hold (ruling 80), so reconcile catches up here: the row is held
+// for the completed op, a RowRetiring hold, until the park-local retires
+// it (Task 10b decision 3: nothing un-holds).
+//
+// In scope only with region HA on, and only on positive evidence on the
+// fresh claim: a serving op on another box (not this one, which serves
+// through Serving) that journals a completed relocation (not completed on
+// this box) and a park-local this box owes for exactly that op, which
+// CompleteRelocation queues for the source and every installed box but
+// the target. A box on lost is left to its park-local (decision 3). A
+// park-local for another op, or none, is out of scope and main's path
+// runs. A completion record that cannot be read is its error, never "no
+// completion": the row is kept and nothing is minted. The path is hold
+// only (Install, checkOn): no activation, no inventory, no execution.
+func (r Reconciler) completedHold(c *hostclaim.Claim) (RowDecision, bool, error) {
+ if !r.relocationAware() || c == nil || c.Holder == nil || r.Box == "" || c.Op.Kind != hostclaim.KindServing || c.Op.Exec == r.Box || slices.Contains(c.Lost, r.Box) {
+ return RowDecision{}, false, nil
+ }
+ done, ok, err := hostclaim.ReadCompletedRelocation(c)
+ if err != nil {
+ return RowDecision{}, false, err
+ }
+ if !ok || done.Stop.Box == r.Box {
+ return RowDecision{}, false, nil
+ }
+ for _, o := range c.Obligations {
+ if o.Kind == "park-local" && o.Box == r.Box && o.Op == done.Op {
+ return RowDecision{Case: RowRetiring, HoldOp: done.Op}, true, nil
+ }
+ }
+ return RowDecision{}, false, nil
+}
+
func (r Reconciler) fenceClosed(ctx context.Context, obs hostclaim.Observed) (bool, error) {
if r.FenceClosed == nil {
return false, nil
@@ -135,11 +177,24 @@ func (r Reconciler) checkOn(ctx context.Context, obs hostclaim.Observed, selecte
return ErrChanged
}
holdOnly := false
- if !r.relocates(obs.Claim) {
+ caught, catchUp, err := r.completedHold(obs.Claim)
+ if err != nil {
+ return err
+ }
+ switch {
+ case catchUp:
+ // The catch-up hold is hold only, on Install's own decision: the
+ // activation reader, the inventory callback and any caller
+ // without that decision are refused before anything is written.
+ if dec == nil || !dec.holdOnly || dec.row != caught {
+ return ErrChanged
+ }
+ holdOnly = true
+ case !r.relocates(obs.Claim):
if err := Serving(obs.Claim, Selection{Box: r.Box}); err != nil {
return err
}
- } else {
+ default:
d := RowDecision{}
if dec != nil {
d, holdOnly = dec.row, dec.holdOnly
@@ -222,6 +277,11 @@ func (r Reconciler) Install(ctx context.Context, selected RecordSelection, write
// With refuses): a kept row's hold only, under the flock. A
// target before installing has no row and mints none (ruling 17).
dec = &decided{row: d, holdOnly: d.Case == RowHeld && (obs.Claim.Op.Exec != r.Box || closed)}
+ } else if d, ok, err := r.completedHold(obs.Claim); err != nil {
+ return err
+ } else if ok {
+ // A completed relocation's catch-up hold: always hold only.
+ dec = &decided{row: d, holdOnly: true}
}
if err := r.checkOn(ctx, obs, selected, dec); err != nil {
return err
@@ -278,12 +338,23 @@ func (r Reconciler) holdOnly(ctx context.Context, selected RecordSelection, d Ro
if err != nil {
return err
}
- if fresh.Absent || fresh.Claim == nil || fresh.Claim.Holder == nil || fresh.Claim.Holder.Incarnation != selected.Incarnation || !r.relocates(fresh.Claim) {
+ if fresh.Absent || fresh.Claim == nil || fresh.Claim.Holder == nil || fresh.Claim.Holder.Incarnation != selected.Incarnation {
return ErrChanged
}
- again, _, err := r.rowDecision(ctx, fresh)
- if err != nil {
+ // The decision is remade in whichever scope the fresh read is in:
+ // the catch-up decision (RowRetiring) and rowDecision's held one
+ // (RowHeld) never compare equal, so a scope change is ErrChanged.
+ again, catchUp, err := r.completedHold(fresh.Claim)
+ switch {
+ case err != nil:
return err
+ case catchUp:
+ case r.relocates(fresh.Claim):
+ if again, _, err = r.rowDecision(ctx, fresh); err != nil {
+ return err
+ }
+ default:
+ return ErrChanged
}
if again != d {
return ErrChanged
diff --git a/internal/hostbox/reconcile_relocation_catchup_test.go b/internal/hostbox/reconcile_relocation_catchup_test.go
new file mode 100644
index 000000000..14d381b95
--- /dev/null
+++ b/internal/hostbox/reconcile_relocation_catchup_test.go
@@ -0,0 +1,251 @@
+//go:build unix
+
+package hostbox
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "slices"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/hostclaim"
+)
+
+// completeFromInstalling is the target finishing after the source's
+// fence closed: its probe passes on "dst", root stops the probe session,
+// and the real serving CAS (hostclaim.CompleteRelocation) queues the
+// source's park-local. No reconcile runs on the source meanwhile.
+func (s *sourceReconcile) completeFromInstalling(t *testing.T) hostclaim.Observed {
+ t.Helper()
+ s.update(t, func(c *hostclaim.Claim) error {
+ h := c.Holder
+ h.Session, h.Generation, h.WriterBox = h.Session+1, "P1", "dst"
+ h.Admissions = append(h.Admissions, hostclaim.AdmissionRecord{Session: h.Session, Box: "dst", Generation: "P1", Predecessor: "g1", Mode: "cold", Region: "ie"})
+ probe, _ := json.Marshal(hostclaim.Probe{Box: "dst", Session: h.Session, At: time.Unix(90, 0).UTC()})
+ stop, _ := json.Marshal(hostclaim.ProbeStop{Box: "dst", Session: h.Session, Receipt: "probe-fence-1", ReceiptSHA256: strings.Repeat("a", 64)})
+ c.Op.Phase = hostclaim.PhaseRelocatingProbing
+ c.Op.Journal["probe"], c.Op.Journal["probe_stop"] = string(probe), string(stop)
+ return nil
+ })
+ n := 0
+ o := s.update(t, hostclaim.CompleteRelocation("rel-1", func() string { n++; return "id-" + strconv.Itoa(n) }))
+ if o.Claim.Op.Kind != hostclaim.KindServing || o.Claim.Op.Exec != "dst" || !hostclaim.OwesLocal(o.Claim, "box") {
+ t.Fatalf("after the completion: op %+v obligations %+v", o.Claim.Op, o.Claim.Obligations)
+ }
+ return o
+}
+
+// L18 joint round, finding 3, through the production ReconcileAdapter,
+// the real census (CheckHeld), park.Local and provision's pass: the
+// source's fence closes in releasing, its reconcile is delayed past
+// installing and the target's completion, and only then does it run. The
+// claim now serves on "dst", so Task 10b's relocating scope is gone; the
+// catch-up hold keeps the row held for the completed op, never ready,
+// with its ownership and every other column as minted, the guard record
+// and the claim unchanged. A replay changes nothing.
+func TestReconcileCatchUpHoldsTheSourceAfterTheTargetCompleted(t *testing.T) {
+ s := newSourceReconcile(t)
+ minted := *s.row(t)
+ s.releasingThenInstalling(t)
+ o := s.completeFromInstalling(t)
+ guard := targetGuardFiles(t, s.f.data)
+ for i := 0; i < 2; i++ {
+ s.pass(t)
+ row := s.row(t)
+ if row == nil || row.Status != "relocating" || row.HoldOp != "rel-1" {
+ t.Fatalf("pass %d after the completion: the source's row is %+v (minted %+v)", i, row, minted)
+ }
+ held := *row
+ held.Status, held.HoldOp = minted.Status, minted.HoldOp
+ if held != minted {
+ t.Fatalf("pass %d: the hold changed more than status and hold_op: %+v", i, row)
+ }
+ if got := targetGuardFiles(t, s.f.data); got[s.f.host+".json"] != guard[s.f.host+".json"] {
+ t.Fatalf("pass %d: the guard record changed", i)
+ }
+ if after, err := hostclaim.Read(t.Context(), s.f.store, s.f.host); err != nil || after.ETag != o.ETag {
+ t.Fatalf("pass %d: the claim was written: %v", i, err)
+ }
+ }
+}
+
+// The catch-up hold through the pass: a row another op holds is never
+// taken, and with region HA off (neither hook set) the pass is main's,
+// the row ready and exactly as minted.
+func TestReconcileCatchUpHoldGuardsThroughThePass(t *testing.T) {
+ t.Run("another op's row", func(t *testing.T) {
+ s := newSourceReconcile(t)
+ if ok, err := s.reg.SetHoldIf(s.f.host, "moving", "mv-9", ""); err != nil || !ok {
+ t.Fatalf("holding for another op: %v %v", ok, err)
+ }
+ before := *s.row(t)
+ s.releasingThenInstalling(t)
+ s.completeFromInstalling(t)
+ s.pass(t)
+ if got := s.row(t); *got != before {
+ t.Fatalf("another op's row was taken: %+v (was %+v)", got, before)
+ }
+ })
+ t.Run("region HA off", func(t *testing.T) {
+ s := newSourceReconcile(t)
+ s.adapter.Reconciler.FenceClosed, s.adapter.Reconciler.HoldRow = nil, nil
+ s.adapter.Configure(&s.cfg)
+ minted := *s.row(t)
+ s.releasingThenInstalling(t)
+ s.completeFromInstalling(t)
+ s.pass(t)
+ if got := s.row(t); *got != minted {
+ t.Fatalf("region HA off: the row is %+v (was %+v)", got, minted)
+ }
+ })
+}
+
+// completedServing rewrites the fixture's claim as a completed relocation's
+// serving op: exec on exec, the completion recorded on stopBox, and the
+// park-local obligations given.
+func completedServing(t *testing.T, r Reconciler, host, exec, stopBox string, owed ...hostclaim.Obligation) hostclaim.Observed {
+ t.Helper()
+ ctx := context.Background()
+ obs, err := hostclaim.Read(ctx, r.Claims, host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ done, err := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-1", Stop: hostclaim.ProbeStop{Box: stopBox, Session: 7, Receipt: "probe-fence-1", ReceiptSHA256: strings.Repeat("a", 64)}, Phase: hostclaim.PhaseRelocatingProbing})
+ if err != nil {
+ t.Fatal(err)
+ }
+ obs, err = hostclaim.Update(ctx, r.Claims, obs, func(c *hostclaim.Claim) error {
+ c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: exec, Participants: []string{exec}, Journal: map[string]string{hostclaim.JournalCompleted: string(done)}}
+ c.Obligations = append(c.Obligations, owed...)
+ return nil
+ })
+ if err != nil {
+ t.Fatal(err)
+ }
+ return obs
+}
+
+func parkLocal(box, op string) hostclaim.Obligation {
+ return hostclaim.Obligation{ID: "ob-" + box + "-" + op, Op: op, Kind: "park-local", Box: box, Retirement: "rt-" + box + "-" + op, Owner: "box:" + box}
+}
+
+// The catch-up hold's scope, guard by guard, on the stubbed reconciler
+// (relocationReconciler, its HoldRow recording): held for the completed
+// op on positive evidence only, hold only, and nowhere else.
+func TestReconcileCatchUpHoldScope(t *testing.T) {
+ ctx := context.Background()
+ install := func(r Reconciler, selected RecordSelection, writes *int) error {
+ return r.Install(ctx, selected, func() error { *writes++; return nil })
+ }
+ t.Run("the source owing the completed op's park-local is held, hold only", func(t *testing.T) {
+ r, selected, holds, _ := relocationReconciler(t)
+ obs := completedServing(t, r, selected.Host, "dst", "dst", parkLocal("box", "rel-1"))
+ writes := 0
+ if err := install(r, selected, &writes); err != nil || writes != 0 || !slices.Equal(*holds, []string{"rel-1"}) {
+ t.Fatalf("the catch-up hold: %v writes %d holds %v", err, writes, *holds)
+ }
+ // No installation authority: the activation reader's and the
+ // inventory callback's check, which carry no Install decision, and
+ // an activation decision, are all refused.
+ if err := r.check(ctx, obs, selected); !errors.Is(err, ErrChanged) {
+ t.Fatalf("the activation path's check under the catch-up: %v", err)
+ }
+ d, ok, err := r.completedHold(obs.Claim)
+ if err != nil || !ok {
+ t.Fatalf("completedHold: %v %v", ok, err)
+ }
+ if err := r.checkOn(ctx, obs, selected, &decided{row: d}); !errors.Is(err, ErrChanged) {
+ t.Fatalf("an activation decision under the catch-up: %v", err)
+ }
+ if err := r.checkOn(ctx, obs, selected, &decided{row: RowDecision{Case: RowHeld, HoldOp: "rel-1"}, holdOnly: true}); !errors.Is(err, ErrChanged) {
+ t.Fatalf("a hold decision the catch-up did not make: %v", err)
+ }
+ })
+ for name, tc := range map[string]struct {
+ exec, stop string
+ owed []hostclaim.Obligation
+ lost bool
+ wantWrites int // main's serving executor writes through activation
+ }{
+ "this box serves": {exec: "box", stop: "dst", owed: []hostclaim.Obligation{parkLocal("box", "rel-1")}, wantWrites: 1},
+ "completed on this box": {exec: "dst", stop: "box", owed: []hostclaim.Obligation{parkLocal("box", "rel-1")}},
+ "no park-local for this box": {exec: "dst", stop: "dst", owed: []hostclaim.Obligation{parkLocal("other", "rel-1")}},
+ "a park-local for another op": {exec: "dst", stop: "dst", owed: []hostclaim.Obligation{parkLocal("box", "rel-0")}},
+ "this box on lost": {exec: "dst", stop: "dst", owed: []hostclaim.Obligation{parkLocal("box", "rel-1")}, lost: true},
+ "another kind of obligation for box": {exec: "dst", stop: "dst", owed: []hostclaim.Obligation{{ID: "ob-1", Op: "rel-1", Kind: "relocation-release", Box: "box", Region: "ie", Retirement: "rt-1", Owner: "region:ie"}}},
+ } {
+ t.Run(name, func(t *testing.T) {
+ r, selected, holds, _ := relocationReconciler(t)
+ completedServing(t, r, selected.Host, tc.exec, tc.stop, tc.owed...)
+ if tc.lost {
+ obs, err := hostclaim.Read(ctx, r.Claims, selected.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := hostclaim.Update(ctx, r.Claims, obs, func(c *hostclaim.Claim) error { c.Lost = append(c.Lost, "box"); return nil }); err != nil {
+ t.Fatal(err)
+ }
+ }
+ writes := 0
+ err := install(r, selected, &writes)
+ if len(*holds) != 0 || writes != tc.wantWrites {
+ t.Fatalf("%v: holds %v writes %d (want %d)", err, *holds, writes, tc.wantWrites)
+ }
+ if tc.wantWrites == 0 && !errors.Is(err, ErrChanged) {
+ t.Fatalf("main's Serving refusal: %v", err)
+ }
+ })
+ }
+ t.Run("region HA off", func(t *testing.T) {
+ r, selected, holds, _ := relocationReconciler(t)
+ r.FenceClosed, r.HoldRow = nil, nil
+ completedServing(t, r, selected.Host, "dst", "dst", parkLocal("box", "rel-1"))
+ writes := 0
+ if err := install(r, selected, &writes); !errors.Is(err, ErrChanged) || writes != 0 || len(*holds) != 0 {
+ t.Fatalf("region HA off: %v writes %d holds %v", err, writes, *holds)
+ }
+ })
+ t.Run("the decision is remade under the lock", func(t *testing.T) {
+ r, selected, _, _ := relocationReconciler(t)
+ completedServing(t, r, selected.Host, "dst", "dst", parkLocal("box", "rel-1"))
+ obs, err := hostclaim.Read(ctx, r.Claims, selected.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ d, ok, err := r.completedHold(obs.Claim)
+ if err != nil || !ok {
+ t.Fatalf("completedHold: %v %v", ok, err)
+ }
+ // The park-local is discharged between the decision and the lock.
+ if _, err := hostclaim.Update(ctx, r.Claims, obs, func(c *hostclaim.Claim) error { c.Obligations = nil; return nil }); err != nil {
+ t.Fatal(err)
+ }
+ if err := r.holdOnly(ctx, selected, d); !errors.Is(err, ErrChanged) {
+ t.Fatalf("a catch-up decision the fresh read no longer makes: %v", err)
+ }
+ })
+}
+
+// RegistryHoldRow's guards hold for the catch-up as they do for Task 10b:
+// a held row for the completed op is a replay with nothing written.
+func TestReconcileCatchUpHoldReplayWritesNothing(t *testing.T) {
+ r, selected, reg, _ := sourceRows(t)
+ completedServing(t, r, selected.Host, "dst", "dst", parkLocal("box", "rel-1"))
+ if err := r.Install(context.Background(), selected, func() error { return errors.New("activated") }); err != nil {
+ t.Fatalf("the catch-up hold: %v", err)
+ }
+ held := *rowOf(t, reg, selected.Host)
+ if held.Status != "relocating" || held.HoldOp != "rel-1" {
+ t.Fatalf("held row %+v", held)
+ }
+ if err := r.Install(context.Background(), selected, func() error { return errors.New("activated") }); err != nil {
+ t.Fatalf("the replay: %v", err)
+ }
+ if got := rowOf(t, reg, selected.Host); *got != held {
+ t.Fatalf("the replay changed the row: %+v (was %+v)", got, held)
+ }
+}
diff --git a/internal/hostbox/relocationsource.go b/internal/hostbox/relocationsource.go
index 6794d9434..1afa278bc 100644
--- a/internal/hostbox/relocationsource.go
+++ b/internal/hostbox/relocationsource.go
@@ -35,7 +35,8 @@ func AdmissionFenceClosed(g hostguard.Guard) func(context.Context, hostclaim.Obs
}
}
-// RegistryHoldRow is Reconciler.HoldRow over this box's registry (R5-3):
+// RegistryHoldRow is Reconciler.HoldRow over this box's registry (R5-3,
+// and the completed relocation's catch-up hold, L18 finding 3):
// it keeps the host's existing row held for op in "relocating", a status
// the edge never serves (registry.HoldStatuses), with registry.SetHoldIf
// expecting the row unheld, the conditional write moves' holdExisting