amadan

carlos / platform Public

host: the hold release refuses a row held by another op before anything is released, discharges again rather than inferring it from a released hold, and answers ErrChanged for a claim that does not serve on this box; N6 is wired whenever claim mode is on; and of the eleven mutations run this round (the review's four survivors, I1, M2, M4, N6's two wiring cases and N6's two fallbacks), all eleven fail a test, one of them only once Plan 11's routing was pinned through N6, where 3a146834 had claimed every guard without the review's four.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude 8f230120150ce145b0984957a4efd88f0ba88a1d parent 3a14683
5 files changed, +373 −60
  • internal/host/claimconstruction.go +10 −10
  • internal/host/claimhandover_crossbox.go +56 −26
  • internal/host/relocationhandover_unix_test.go +206 −0
  • internal/host/relocationowners.go +26 −10
  • internal/host/relocationreleasehold_test.go +75 −14

CI cancelled — superseded by a newer push — run details

diff --git a/internal/host/claimconstruction.go b/internal/host/claimconstruction.go
index 514de5704..ea264a84b 100644
--- a/internal/host/claimconstruction.go
+++ b/internal/host/claimconstruction.go
@@ -281,17 +281,17 @@ func OpenProductionClaimAssembly(ctx context.Context, store release.Store, r Cla
}
wake.AfterAdmission = coldAdmissionHook(guard, owners.Identity, controlsock.Path)
wake.RetireArtifacts = func(context.Context, hostclaim.Observed, ClaimLaunchInputs, string) error { return hostbox.ErrChanged }
+ // Region HA N6 is wired whenever claim mode is on, whatever
+ // CARLOS_REGION_HA says (ruling 82's reasoning): it serves a relocation
+ // that already completed, and without a completion record it does nothing.
+ completed := &relocationCompletedHandover{Guard: guard, Identity: owners.Identity, Socket: controlsock.Path, Box: actor.Box}
+ handover := claimCrossBoxHandover{Guard: guard, Evidence: store, Box: actor.Box}
+ var relocation *ClaimRelocationOwners
+ if movesC1 && groupD.Dispatch.Relocation != nil {
+ relocation = &owners.Relocation
+ }
+ wireProductionAdmission(wake, actor.Box, movesC1, handover, relocation, completed)
if movesC1 {
- handover := claimCrossBoxHandover{Guard: guard, Evidence: store, Box: actor.Box}
- var relocation *ClaimRelocationOwners
- if groupD.Dispatch.Relocation != nil {
- relocation = &owners.Relocation
- }
- var completed *relocationCompletedHandover
- if relocation != nil {
- completed = &relocationCompletedHandover{Guard: guard, Identity: owners.Identity, Socket: controlsock.Path, Box: actor.Box}
- }
- wireClaimAdmission(wake, actor.Box, wake.AfterAdmission, handover, relocation, completed)
retirement := park.Local{Authorize: func(ctx context.Context, c *hostclaim.Claim, o hostclaim.Obligation) error {
return owners.Authorize(ctx, hostclaim.Observed{Host: c.Host, Claim: c}, o)
}, Guard: guard, Claims: store, DataDir: r.DataDir, Box: actor.Box}
diff --git a/internal/host/claimhandover_crossbox.go b/internal/host/claimhandover_crossbox.go
index e2eba8945..7cec26268 100644
--- a/internal/host/claimhandover_crossbox.go
+++ b/internal/host/claimhandover_crossbox.go
@@ -140,40 +140,70 @@ func claimPredecessorStop(w *ClaimWakeAssembly, box string, crossBox func(contex
// wireClaimAdmission routes the source proof to its operation owner before
// lease acquisition and checks the target installation after admission.
+// With completed set, region HA's N6 handover goes in front of it
+// (wireCompletedHandover).
func wireClaimAdmission(w *ClaimWakeAssembly, box string, local func(context.Context, hostclaim.Observed, hostclaim.Observed) error, landing claimCrossBoxHandover, relocation *ClaimRelocationOwners, completed *relocationCompletedHandover) {
before := landing.Before
after := landing.After
- if relocation != nil && completed != nil {
- // Region HA N6: the first serving wake after a relocation completed
- // on this box hands over from the probe's existing receipt; every
- // other serving prior keeps main's stop and local handover. A
- // serving prior on this box whose completion record cannot be read
- // is refused, never routed to main's stop (absence is never
- // evidence): that stop would replace the probe's fence.
- completedAfter := completed.After()
- w.BeforeAdmission = func(ctx context.Context, prior hostclaim.Observed) error {
- if _, ok, err := completed.completedPrior(prior); err != nil {
- return err
- } else if ok {
- return completed.Before(ctx, prior)
- }
- return claimPredecessorStop(w, box, relocationBefore(landing, relocation, box))(ctx, prior)
- }
- w.AfterAdmission = func(ctx context.Context, prior, admitted hostclaim.Observed) error {
- if _, ok, err := completed.completedPrior(prior); err != nil {
- return err
- } else if ok {
- return completedAfter(ctx, prior, admitted)
- }
- return claimAdmissionHandover(box, local, relocationAfter(landing, relocation, box))(ctx, prior, admitted)
- }
- return
- }
if relocation != nil {
before, after = relocationBefore(landing, relocation, box), relocationAfter(landing, relocation, box)
}
w.BeforeAdmission = claimPredecessorStop(w, box, before)
w.AfterAdmission = claimAdmissionHandover(box, local, after)
+ if completed != nil {
+ wireCompletedHandover(w, completed)
+ }
+}
+
+// wireCompletedHandover is region HA N6 in front of whatever admission
+// hooks w already has: the first serving wake after a relocation completed
+// on this box hands over from the probe's existing receipt; every other
+// prior keeps those hooks unchanged (with no BeforeAdmission, main's
+// Wake.Stop). It is wired whenever claim mode is on, not only with region
+// HA on (ruling 82's reasoning, Task 9 review): it serves a relocation
+// that has already completed, and without a completion record it does
+// nothing. A serving prior on this box whose completion record cannot be
+// read is refused, never routed to main's stop (absence is never
+// evidence): that stop would replace the probe's fence.
+func wireCompletedHandover(w *ClaimWakeAssembly, completed *relocationCompletedHandover) {
+ before, after := w.BeforeAdmission, w.AfterAdmission
+ completedAfter := completed.After()
+ w.BeforeAdmission = func(ctx context.Context, prior hostclaim.Observed) error {
+ if _, ok, err := completed.completedPrior(prior); err != nil {
+ return err
+ } else if ok {
+ return completed.Before(ctx, prior)
+ }
+ if before != nil {
+ return before(ctx, prior)
+ }
+ return w.Stop(ctx, prior)
+ }
+ w.AfterAdmission = func(ctx context.Context, prior, admitted hostclaim.Observed) error {
+ if _, ok, err := completed.completedPrior(prior); err != nil {
+ return err
+ } else if ok {
+ return completedAfter(ctx, prior, admitted)
+ }
+ if after == nil {
+ return hostbox.ErrChanged
+ }
+ return after(ctx, prior, admitted)
+ }
+}
+
+// wireProductionAdmission is OpenProductionClaimAssembly's admission
+// wiring: with moves C1 on, wireClaimAdmission over the cross-box landing
+// handover and, with region HA's owners bound, the relocating routing;
+// with it off, main's hooks as they are. N6 goes in front in both, so a
+// relocation completed while region HA was on is handed over from its
+// probe's receipt at the first serving wake after region HA is turned off.
+func wireProductionAdmission(w *ClaimWakeAssembly, box string, movesC1 bool, landing claimCrossBoxHandover, relocation *ClaimRelocationOwners, completed *relocationCompletedHandover) {
+ if movesC1 {
+ wireClaimAdmission(w, box, w.AfterAdmission, landing, relocation, completed)
+ return
+ }
+ wireCompletedHandover(w, completed)
}
func claimRemotePredecessor(prior hostclaim.Observed, box string) bool {
diff --git a/internal/host/relocationhandover_unix_test.go b/internal/host/relocationhandover_unix_test.go
index 2cf5288cc..f3ff3dafb 100644
--- a/internal/host/relocationhandover_unix_test.go
+++ b/internal/host/relocationhandover_unix_test.go
@@ -8,13 +8,17 @@ import (
"encoding/hex"
"encoding/json"
"errors"
+ "fmt"
"net"
"net/http"
"path/filepath"
+ "slices"
"testing"
"github.com/carlosframework/platform/internal/hostbox"
"github.com/carlosframework/platform/internal/hostclaim"
+ execlease "github.com/carlosframework/platform/internal/hostexec"
+ "github.com/carlosframework/platform/internal/release"
)
// N6, ruling 56: wireClaimAdmission with the completed-relocation branch.
@@ -132,3 +136,205 @@ func TestRelocationCompletedHandoverRoutesOnlyTheCompletedPrior(t *testing.T) {
t.Fatalf("an unreadable record after admission: %v (locals %d, posted %d)", err, locals, len(posted))
}
}
+
+// handoverRoot is root's /claim/handover endpoint on a unix socket,
+// recording every request it is posted.
+func handoverRoot(t *testing.T) (string, *[]coldHandoverRequest) {
+ t.Helper()
+ socket := filepath.Join(t.TempDir(), "c.sock")
+ l, err := net.Listen("unix", socket)
+ if err != nil {
+ t.Fatal(err)
+ }
+ posted := &[]coldHandoverRequest{}
+ srv := &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ var req coldHandoverRequest
+ if r.URL.Path != "/claim/handover" || json.NewDecoder(r.Body).Decode(&req) != nil {
+ w.WriteHeader(http.StatusBadRequest)
+ return
+ }
+ *posted = append(*posted, req)
+ w.WriteHeader(http.StatusNoContent)
+ })}
+ go srv.Serve(l)
+ t.Cleanup(func() { srv.Close() })
+ return socket, posted
+}
+
+// completedServing is the serving op CompleteRelocation leaves after the
+// probe stop s of the fixture's relocating claim: the same holder, and
+// the completed relocation journaled, marked discharged when the
+// completion pass has run.
+func (f *stopProbeFixture) completedServing(s hostclaim.ProbeStop, discharged bool) hostclaim.Observed {
+ f.t.Helper()
+ c := *f.obs.Claim
+ c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"},
+ Journal: map[string]string{hostclaim.JournalCompleted: string(mustJSON(f.t, hostclaim.CompletedRelocation{Op: "rel-1", Stop: s, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"}))}}
+ if discharged {
+ if err := hostclaim.MarkCompletedDischarged("rel-1")(&c); err != nil {
+ f.t.Fatal(err)
+ }
+ }
+ return hostclaim.Observed{Host: f.obs.Host, ETag: "e2", Claim: &c}
+}
+
+// Ruling 82's reasoning, applied to N6 (Task 9 review): a relocation
+// completed while region HA was on, and region HA is now off. The claim
+// assembly then has no relocation owners (and, with moves C1 off too, no
+// cross-box routing at all), yet the first serving wake must still hand
+// over from the probe's receipt: main's Wake.Stop would ask a
+// serving-shaped re-stop of the stopped probe session. Through
+// OpenProductionClaimAssembly's own wiring (wireProductionAdmission), the
+// wake's predecessor stop finds and verifies the receipt with nothing
+// stopped, and root is posted that receipt's target as the prior. A prior
+// with no completion record, on a box that completed nothing, meets main's
+// hooks exactly as they were.
+func TestRelocationCompletedHandoverWithRegionHAOff(t *testing.T) {
+ ctx := t.Context()
+ for _, movesC1 := range []bool{true, false} {
+ for _, discharged := range []bool{false, true} {
+ t.Run(fmt.Sprintf("moves C1 %v, discharged %v", movesC1, discharged), func(t *testing.T) {
+ f := newStopProbeFixture(t)
+ s, err := f.stop()
+ if err != nil {
+ t.Fatal(err)
+ }
+ f.order = nil
+ socket, posted := handoverRoot(t)
+ stops, locals := 0, 0
+ w := &ClaimWakeAssembly{Stop: func(context.Context, hostclaim.Observed) error { stops++; return nil }}
+ w.AfterAdmission = func(context.Context, hostclaim.Observed, hostclaim.Observed) error { locals++; return nil }
+ identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: "dst", Enrollment: "enr", Region: "ie", Revision: 1})
+ wireProductionAdmission(w, "dst", movesC1, claimCrossBoxHandover{Box: "dst"}, nil,
+ &relocationCompletedHandover{Guard: f.a.Guard, Identity: identity, Socket: socket, Box: "dst"})
+
+ prior := f.completedServing(s, discharged)
+ admitted := *prior.Claim
+ next := *admitted.Holder
+ next.Stamp, next.Session, next.Generation = 40, 40, "g40"
+ admitted.Holder = &next
+ if err := w.stopPredecessor(ctx, prior); err != nil || stops != 0 || len(f.order) != 0 {
+ t.Fatalf("the first serving wake's predecessor stop: %v (stops %d, root %v)", err, stops, f.order)
+ }
+ if err := w.AfterAdmission(ctx, prior, hostclaim.Observed{Host: prior.Host, ETag: "e3", Claim: &admitted}); err != nil {
+ t.Fatalf("the first serving wake's handover: %v", err)
+ }
+ if len(*posted) != 1 || (*posted)[0].Prior != f.target || (*posted)[0].Next.Session != 40 || locals != 0 {
+ t.Fatalf("root was posted %+v (locals %d)", *posted, locals)
+ }
+ })
+ }
+ t.Run(fmt.Sprintf("moves C1 %v, no completion record", movesC1), func(t *testing.T) {
+ // A box that never completed a relocation: an empty guard
+ // journal, a serving prior with no record.
+ f := newStopProbeFixture(t)
+ socket, posted := handoverRoot(t)
+ stops, locals := 0, 0
+ w := &ClaimWakeAssembly{Stop: func(context.Context, hostclaim.Observed) error { stops++; return nil }}
+ w.AfterAdmission = func(context.Context, hostclaim.Observed, hostclaim.Observed) error { locals++; return nil }
+ identity := handoverRecoveryIdentity(t, hostbox.Actor{Box: "dst", Enrollment: "enr", Region: "ie", Revision: 1})
+ wireProductionAdmission(w, "dst", movesC1, claimCrossBoxHandover{Box: "dst"}, nil,
+ &relocationCompletedHandover{Guard: f.a.Guard, Identity: identity, Socket: socket, Box: "dst"})
+ c := *f.obs.Claim
+ c.Op = hostclaim.Op{Kind: hostclaim.KindServing, Exec: "dst", Participants: []string{"dst"}}
+ prior := hostclaim.Observed{Host: f.obs.Host, ETag: "e2", Claim: &c}
+ if err := w.stopPredecessor(ctx, prior); err != nil || stops != 1 {
+ t.Fatalf("main's stop: %v (stops %d)", err, stops)
+ }
+ if err := w.AfterAdmission(ctx, prior, prior); err != nil || locals != 1 || len(*posted) != 0 {
+ t.Fatalf("main's handover: %v (locals %d, posted %d)", err, locals, len(*posted))
+ }
+ })
+ }
+}
+
+// C10 and N6's Before verify root's receipt, not only find it: a receipt
+// in the guard journal whose fence id, phase and digest match the serving
+// op's record, but whose bytes disagree with their own fence target, is
+// refused by both. With root's genuine receipt both pass.
+func TestRelocationCompletedReceiptIsVerifiedByC10AndN6(t *testing.T) {
+ ctx := t.Context()
+ for _, genuine := range []bool{true, false} {
+ t.Run(fmt.Sprintf("genuine %v", genuine), func(t *testing.T) {
+ f := newStopProbeFixture(t)
+ if !genuine {
+ var r StopReceipt
+ if err := json.Unmarshal(f.receipt, &r); err != nil {
+ t.Fatal(err)
+ }
+ r.BootID = "another-boot" // the target says "boot"
+ f.receipt = mustMarshalStopReceipt(r)
+ }
+ s, err := f.stop()
+ if err != nil {
+ t.Fatal(err)
+ }
+ prior := f.completedServing(s, false)
+ check := func(what string, err error) {
+ t.Helper()
+ if genuine && err != nil {
+ t.Fatalf("%s refused root's receipt: %v", what, err)
+ }
+ if !genuine && !errors.Is(err, ErrClaimStopReceipt) {
+ t.Fatalf("%s accepted a receipt that does not verify: %v", what, err)
+ }
+ }
+ h := relocationCompletedHandover{Guard: f.a.Guard, Box: "dst"}
+ check("N6's Before", h.Before(ctx, prior))
+
+ // C10 over the persisted binding, the serving claim in the store.
+ store := release.NewDirStore(t.TempDir())
+ key, _ := hostclaim.Key(prior.Host)
+ if _, err := store.Put(ctx, key, mustJSON(t, prior.Claim), ""); err != nil {
+ t.Fatal(err)
+ }
+ f.a.Claims = store
+ done, err := f.a.completedRelocationDischarge(ctx, execlease.Binding{Box: "dst", Host: prior.Host, Incarnation: prior.Claim.Holder.Incarnation, Op: "rel-1"})
+ if !done {
+ t.Fatalf("C10 fell through to main's cleanup: %v", err)
+ }
+ check("C10", err)
+ })
+ }
+}
+
+// N6 is now always in front of the admission hooks in claim mode, so
+// Plan 11's relocating routing reaches its owners through N6's fallback:
+// TestPlan11Fix1RelocationAdmissionHooks's three writers, with N6 wired as
+// production wires it.
+func TestRelocationCompletedHandoverKeepsPlan11Routing(t *testing.T) {
+ for _, writer := range []string{"source", "", "target"} {
+ t.Run(writer, func(t *testing.T) {
+ calls := []string{}
+ w := &ClaimWakeAssembly{Stop: func(context.Context, hostclaim.Observed) error { calls = append(calls, "local-stop"); return nil }}
+ w.AfterAdmission = func(context.Context, hostclaim.Observed, hostclaim.Observed) error {
+ calls = append(calls, "local-after")
+ return nil
+ }
+ reloc := ClaimRelocationOwners{BeforeAdmission: func(context.Context, hostclaim.Observed) error {
+ calls = append(calls, "relocation-before")
+ return nil
+ }, AfterAdmission: func(context.Context, hostclaim.Observed, hostclaim.Observed) error {
+ calls = append(calls, "relocation-after")
+ return nil
+ }}
+ wireProductionAdmission(w, "target", true, claimCrossBoxHandover{}, &reloc, &relocationCompletedHandover{Box: "target"})
+ o := plan11Relocating("probing")
+ o.Claim.Op.Exec = "target"
+ o.Claim.Holder.WriterBox = writer
+ if err := w.stopPredecessor(t.Context(), o); err != nil {
+ t.Fatal(err)
+ }
+ if err := w.AfterAdmission(t.Context(), o, o); err != nil {
+ t.Fatal(err)
+ }
+ want := []string{"relocation-before", "relocation-after"}
+ if writer == "target" {
+ want = []string{"local-stop", "local-after"}
+ }
+ if !slices.Equal(calls, want) {
+ t.Fatalf("calls=%v want=%v", calls, want)
+ }
+ })
+ }
+}
diff --git a/internal/host/relocationowners.go b/internal/host/relocationowners.go
index 286472639..3ef8cc4b8 100644
--- a/internal/host/relocationowners.go
+++ b/internal/host/relocationowners.go
@@ -403,15 +403,21 @@ func readRelocationProbeFence(path string) (relocationProbeFence, bool, error) {
// completion record names this box and is not yet marked, never from the
// row (ruling 80), and every step is idempotent, so a failure or a crash
// at any point leaves the record unmarked and the next pass resumes where
-// it stopped. Once the row is no longer held for the op, the discharge has
-// happened (nothing else releases a relocating hold for a completed op),
-// and a replay only lifts and marks. A missing row is refused: absence is
-// never evidence of the discharge. So is a missing, unreadable or
-// mismatched probe-fence owner, read before anything is released: the
-// probe stop persisted it before it fenced, and without it nothing can
-// say the fence is lifted, so the mark is never written over its absence.
-// Anything else leaves everything: an excluded box's row is its
-// park-local's.
+// it stopped. Once the row's hold is released, a replay discharges again
+// rather than inferring that the discharge ran (absence is never
+// evidence): main's discharge then finds no slot and no .hostexec record,
+// which is positive evidence, and does nothing more; only then are the
+// fence lifted and the record marked. A row held by any other op is
+// refused with nothing done: nothing has proved this op's participation
+// discharged. A missing row is refused: absence is never evidence of the
+// discharge. So is a missing, unreadable or mismatched probe-fence owner,
+// read before anything is released: the probe stop persisted it before it
+// fenced, and without it nothing can say the fence is lifted, so the mark
+// is never written over its absence. A claim that is absent, or does not
+// serve on this box, is ErrChanged: the pass asks only about completions
+// recorded on this box (ruling 80), so a caller that asks about anything
+// else is mis-wired, never finished. Anything else leaves everything: an
+// excluded box's row is its park-local's.
func (a ClaimAssembly) relocationReleaseHold(cfg RelocationAssemblyConfig) func(context.Context, string, string) error {
return func(ctx context.Context, host, op string) error {
if cfg.Registry == nil || cfg.Activator == nil || cfg.Discharge == nil {
@@ -426,7 +432,7 @@ func (a ClaimAssembly) relocationReleaseHold(cfg RelocationAssemblyConfig) func(
return err
}
if obs.Absent || obs.Claim == nil || obs.Claim.Holder == nil || obs.Claim.Op.Kind != hostclaim.KindServing || obs.Claim.Op.Exec != local.Actor.Box || obs.Claim.Holder.WriterBox != local.Actor.Box {
- return nil
+ return fmt.Errorf("%w: %s does not serve on this box", hostbox.ErrChanged, host)
}
done, ok, err := hostclaim.ReadCompletedRelocation(obs.Claim)
if err != nil {
@@ -445,6 +451,9 @@ func (a ClaimAssembly) relocationReleaseHold(cfg RelocationAssemblyConfig) func(
if row == nil {
return fmt.Errorf("%w: %s has no row on this box", hostrelocate.ErrRefused, host)
}
+ if row.HoldOp != "" && row.HoldOp != op {
+ return fmt.Errorf("%w: %s's row is held by %s, not %s", hostrelocate.ErrRefused, host, row.HoldOp, op)
+ }
marker, err := relocationProbeFencePath(cfg.DataDir, op)
if err != nil {
return err
@@ -466,6 +475,13 @@ func (a ClaimAssembly) relocationReleaseHold(cfg RelocationAssemblyConfig) func(
if _, err := cfg.Registry.ReleaseHold(host, op); err != nil {
return err
}
+ } else if err := cfg.Discharge(ctx, obs); err != nil {
+ // The hold is already released. The holder may have moved on
+ // since, so there is no holder check here: main's serving Check
+ // let no wake through while a slot or record remained, and the
+ // discharge's postcondition (no slot, no record) is the
+ // evidence, not the released hold.
+ return err
}
cfg.Activator.LiftFenceAs(host, fence.Session, fence.Owner)
_, err = hostclaim.Update(ctx, a.Claims, obs, hostclaim.MarkCompletedDischarged(op))
diff --git a/internal/host/relocationreleasehold_test.go b/internal/host/relocationreleasehold_test.go
index cdbf1cdd4..aadd722fc 100644
--- a/internal/host/relocationreleasehold_test.go
+++ b/internal/host/relocationreleasehold_test.go
@@ -29,8 +29,11 @@ import (
// it, so the edge keeps the row held, the activator wakes nothing and no
// successor starts; a holder moved off the probe session while the row is
// held is refused; another operation's fence is never lifted; a replay
-// after the hold went only lifts and marks; a claim that does not serve on
-// this box is left alone.
+// after the hold went discharges again (positive evidence, never the
+// released hold's inference), then lifts and marks; a row held by another
+// op, or a record whose stop was on another box, is refused with nothing
+// done; a claim that does not serve on this box is left alone and is
+// ErrChanged.
func TestRelocationReleaseHoldWaitsForDischarge(t *testing.T) {
const host, box = "shop.example", "dst"
ctx := t.Context()
@@ -58,7 +61,10 @@ func TestRelocationReleaseHoldWaitsForDischarge(t *testing.T) {
owner string
marker string
discharge error
- hold func(context.Context, string, string) error
+ // heldAt is, per discharge, whether the row was still held for
+ // rel-1 when it ran.
+ heldAt []bool
+ hold func(context.Context, string, string) error
}
setup := func(t *testing.T, c *hostclaim.Claim) *world {
t.Helper()
@@ -91,13 +97,14 @@ func TestRelocationReleaseHoldWaitsForDischarge(t *testing.T) {
if _, err := w.act.svc.FenceAndStopAs(fctx, host, 12, w.owner); err != nil && !errors.Is(err, activator.ErrStopUnproven) {
t.Fatal(err)
}
- a := ClaimAssembly{Claims: w.store, stopLocal: func(context.Context) (claimStopLocal, error) {
+ // The claim writes go through markStore, so the mark has its place
+ // in the order.
+ a := ClaimAssembly{Claims: markStore{Store: w.store, order: &w.order}, stopLocal: func(context.Context) (claimStopLocal, error) {
return claimStopLocal{Actor: hostbox.Actor{Box: box}}, nil
}}
cfg := RelocationAssemblyConfig{Registry: w.reg, Activator: w.act, DataDir: dataDir, Discharge: func(_ context.Context, o hostclaim.Observed) error {
- if row, _ := w.reg.Get(host); row == nil || row.HoldOp != "rel-1" {
- t.Error("the hold was released before the discharge")
- }
+ row, _ := w.reg.Get(host)
+ w.heldAt = append(w.heldAt, row != nil && row.HoldOp == "rel-1")
w.order = append(w.order, "discharge")
return w.discharge
}}
@@ -124,8 +131,8 @@ func TestRelocationReleaseHoldWaitsForDischarge(t *testing.T) {
if err := w.hold(ctx, host, "rel-1"); err != nil {
t.Fatal(err)
}
- if want := []string{"discharge", "lift"}; !slices.Equal(w.order, want) || held(w) || w.act.svc.Fenced(host) || !marked(w) {
- t.Fatalf("order %v held=%v fenced=%v marked=%v", w.order, held(w), w.act.svc.Fenced(host), marked(w))
+ if want := []string{"discharge", "lift", "mark"}; !slices.Equal(w.order, want) || !slices.Equal(w.heldAt, []bool{true}) || held(w) || w.act.svc.Fenced(host) || !marked(w) {
+ t.Fatalf("order %v held at discharge %v held=%v fenced=%v marked=%v", w.order, w.heldAt, held(w), w.act.svc.Fenced(host), marked(w))
}
// A replay of a marked relocation does nothing at all.
w.order = nil
@@ -143,6 +150,24 @@ func TestRelocationReleaseHoldWaitsForDischarge(t *testing.T) {
t.Fatalf("order %v held=%v fence=%d marked=%v", w.order, held(w), w.act.svc.FenceGeneration(host), marked(w))
}
})
+ t.Run("a row held by another op is refused with nothing done", func(t *testing.T) {
+ w := setup(t, claim(nil))
+ if ok, err := w.reg.HandHold(host, "relocating", "rel-9", "rel-1"); err != nil || !ok {
+ t.Fatalf("hand the hold to rel-9: %v %v", ok, err)
+ }
+ err := w.hold(ctx, host, "rel-1")
+ row, _ := w.reg.Get(host)
+ if !errors.Is(err, hostrelocate.ErrRefused) || len(w.order) != 0 || row == nil || row.HoldOp != "rel-9" || w.act.svc.FenceGeneration(host) != 12 || marked(w) {
+ t.Fatalf("%v order %v row %+v fence=%d marked=%v", err, w.order, row, w.act.svc.FenceGeneration(host), marked(w))
+ }
+ })
+ t.Run("a record whose stop was on another box is refused with nothing done", func(t *testing.T) {
+ other, _ := json.Marshal(hostclaim.CompletedRelocation{Op: "rel-1", Stop: hostclaim.ProbeStop{Box: "src", Session: 12, Receipt: "fence-1", ReceiptSHA256: strings.Repeat("a", 64)}, Stamp: 9, Phase: hostclaim.PhaseRelocatingProbing, Generation: "P1"})
+ w := setup(t, claim(func(c *hostclaim.Claim) { c.Op.Journal[hostclaim.JournalCompleted] = string(other) }))
+ if err := w.hold(ctx, host, "rel-1"); !errors.Is(err, hostrelocate.ErrRefused) || len(w.order) != 0 || !held(w) || w.act.svc.FenceGeneration(host) != 12 || marked(w) {
+ t.Fatalf("%v order %v held=%v fence=%d marked=%v", err, w.order, held(w), w.act.svc.FenceGeneration(host), marked(w))
+ }
+ })
t.Run("another operation's fence after a restart stays", func(t *testing.T) {
w := setup(t, claim(nil))
w.act.svc = &activator.Service{} // the restart: this box's fence is gone
@@ -154,9 +179,10 @@ func TestRelocationReleaseHoldWaitsForDischarge(t *testing.T) {
t.Fatalf("held=%v fence=%d marked=%v", held(w), w.act.svc.FenceGeneration(host), marked(w))
}
})
- t.Run("a replay after the hold went only lifts and marks", func(t *testing.T) {
+ t.Run("a replay after the hold went discharges, lifts and marks", func(t *testing.T) {
// The hold was released, then the agent stopped before the lift;
- // a serving wake has since moved the holder on.
+ // a serving wake has since moved the holder on. The discharge runs
+ // again, without the holder check, and its answer is the evidence.
w := setup(t, claim(func(c *hostclaim.Claim) { c.Holder.Session = 13 }))
if _, err := w.reg.ReleaseHold(host, "rel-1"); err != nil {
t.Fatal(err)
@@ -164,8 +190,21 @@ func TestRelocationReleaseHoldWaitsForDischarge(t *testing.T) {
if err := w.hold(ctx, host, "rel-1"); err != nil {
t.Fatal(err)
}
- if want := []string{"lift"}; !slices.Equal(w.order, want) || w.act.svc.Fenced(host) || !marked(w) {
- t.Fatalf("order %v fenced=%v marked=%v", w.order, w.act.svc.Fenced(host), marked(w))
+ if want := []string{"discharge", "lift", "mark"}; !slices.Equal(w.order, want) || !slices.Equal(w.heldAt, []bool{false}) || w.act.svc.Fenced(host) || !marked(w) {
+ t.Fatalf("order %v held at discharge %v fenced=%v marked=%v", w.order, w.heldAt, w.act.svc.Fenced(host), marked(w))
+ }
+ })
+ t.Run("a replay after the hold went with a refused discharge keeps the fence and the record", func(t *testing.T) {
+ w := setup(t, claim(nil))
+ if _, err := w.reg.ReleaseHold(host, "rel-1"); err != nil {
+ t.Fatal(err)
+ }
+ w.discharge = errors.New("the participation slot is still open")
+ if err := w.hold(ctx, host, "rel-1"); !errors.Is(err, w.discharge) {
+ t.Fatalf("got %v", err)
+ }
+ if want := []string{"discharge"}; !slices.Equal(w.order, want) || w.act.svc.FenceGeneration(host) != 12 || marked(w) {
+ t.Fatalf("order %v fence=%d marked=%v", w.order, w.act.svc.FenceGeneration(host), marked(w))
}
})
t.Run("a holder moved off the probe session while held", func(t *testing.T) {
@@ -194,11 +233,21 @@ func TestRelocationReleaseHoldWaitsForDischarge(t *testing.T) {
} {
t.Run(name, func(t *testing.T) {
w := setup(t, claim(change))
- if err := w.hold(ctx, host, "rel-1"); err != nil || len(w.order) != 0 || !held(w) || w.act.svc.FenceGeneration(host) != 12 {
+ if err := w.hold(ctx, host, "rel-1"); !errors.Is(err, hostbox.ErrChanged) || len(w.order) != 0 || !held(w) || w.act.svc.FenceGeneration(host) != 12 {
t.Fatalf("%v order %v held=%v", err, w.order, held(w))
}
})
}
+ t.Run("an absent claim", func(t *testing.T) {
+ w := setup(t, claim(nil))
+ key, _ := hostclaim.Key(host)
+ if err := w.store.Delete(ctx, key); err != nil {
+ t.Fatal(err)
+ }
+ if err := w.hold(ctx, host, "rel-1"); !errors.Is(err, hostbox.ErrChanged) || len(w.order) != 0 || !held(w) || w.act.svc.FenceGeneration(host) != 12 {
+ t.Fatalf("%v order %v held=%v", err, w.order, held(w))
+ }
+ })
// Absence is never evidence (the Global Constraint): the hold release
// reads its evidence before it releases anything, and without it
// refuses with nothing done: the row stays held, the probe's fence
@@ -314,3 +363,15 @@ func TestRelocationSuccessorStartsOnlyAfterTheTargetsDischarge(t *testing.T) {
t.Fatalf("the successor after the discharge: %v", err)
}
}
+
+// markStore records every claim write in order as "mark": the hold
+// release writes the claim once, to mark the completion discharged.
+type markStore struct {
+ release.Store
+ order *[]string
+}
+
+func (s markStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) {
+ *s.order = append(*s.order, "mark")
+ return s.Store.Put(ctx, key, body, ifMatch)
+}