amadan

carlos / platform Public

hostclaim: relocating within one region, released by stop evidence or a termination fence, with released coverage on P's one opening and the fenced box's loss retained

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Paul Campbell pushed by claude 54bb6da57a42468afddde56ea01fcaa23f2a6a68 parent 9240bf7
4 files changed, +2415 −0
  • internal/hostclaim/claim.go +11 −0
  • internal/hostclaim/generation.go +65 −0
  • internal/hostclaim/relocation.go +1241 −0
  • internal/hostclaim/relocation_test.go +1098 −0
diff --git a/internal/hostclaim/claim.go b/internal/hostclaim/claim.go
index 25c71ee4d..56a42a91a 100644
--- a/internal/hostclaim/claim.go
+++ b/internal/hostclaim/claim.go
@@ -166,6 +166,7 @@ type Claim struct {
Lost []string `json:"lost,omitempty"`
LostRevisions map[string]uint64 `json:"lost_revisions,omitempty"`
LossRefs []string `json:"loss_refs,omitempty"`
+ EvidenceRefs []AutomaticLoss `json:"evidence_refs,omitempty"`
RetiredIncarnations []string `json:"retired_incarnations,omitempty"`
QuietAfter map[string]time.Time `json:"quiet_after,omitempty"`
CancelledCreates []CancelledCreate `json:"cancelled_creates,omitempty"`
@@ -293,6 +294,16 @@ func (c *Claim) Validate() error {
}
seenLossRefs[id] = true
}
+ seenEvidence := map[string]bool{}
+ for _, l := range c.EvidenceRefs {
+ // D4: a retained automatic loss names a loss identity the claim
+ // also lists, a box, and a well-formed fence reference.
+ if !seenLossRefs[l.ID] || seenEvidence[l.ID] || l.Box == "" || !validFenceRef(l.Fence) || l.Enrollment != l.Fence.Enrollment || l.Op == "" ||
+ (l.Role != "source" && l.Role != "target") || (l.Timers != TimersComplete && l.Timers != TimersPartial && l.Timers != TimersUnknown) {
+ return fmt.Errorf("%w: invalid automatic loss evidence", ErrMalformed)
+ }
+ seenEvidence[l.ID] = true
+ }
if len(c.RetiredIncarnations) > retiredCap {
return fmt.Errorf("%w: retired_incarnations over cap", ErrMalformed)
}
diff --git a/internal/hostclaim/generation.go b/internal/hostclaim/generation.go
index ffd7e058b..74947b568 100644
--- a/internal/hostclaim/generation.go
+++ b/internal/hostclaim/generation.go
@@ -43,6 +43,10 @@ func AdmitWriter(in Admission) func(*Claim) error {
switch c.Op.Kind {
case KindCreating, KindDeleting, KindParking, KindPurging, KindPlacing:
return fmt.Errorf("%w: op kind %q cannot admit a writer", ErrWrongState, c.Op.Kind)
+ case KindRelocating:
+ if err := relocationAdmission(c, in); err != nil {
+ return err
+ }
}
h := c.Holder
if in.LossID != "" && !in.lossVerified {
@@ -87,6 +91,10 @@ func AdmitWriter(in Admission) func(*Claim) error {
switch in.Coverage {
case "", "sealed", "clean-return":
+ case "released": // checked by relocationAdmission (L9)
+ if c.Op.Kind != KindRelocating {
+ return ErrWrongState
+ }
default:
return ErrWrongState
}
@@ -261,3 +269,60 @@ func CoverParkedPredecessors(observedOpID, observedPhase string, region string,
return nil
}
}
+
+// relocationAdmission is §4's phase table for relocating (L1) and §6.4a's
+// `released` coverage (L9). The source may admit in starting and releasing
+// (its local admission fence closes releasing); only op.exec admits in
+// probing, cold; nobody admits in any other phase. The first probe
+// admission opens the prepared generation P, which is opened at most once.
+// `released` is legal only on that opening, when the predecessor is exactly
+// the session whose release this operation journaled and its pin reaches
+// that release's final position.
+func relocationAdmission(c *Claim, in Admission) error {
+ if c.Op.Journal["stuck"] != "" {
+ // Ruling 9: a stuck operation runs nothing, the source's wake
+ // included; the operator resolves it (carlos ops host resolve).
+ return fmt.Errorf("%w: relocation is stuck", ErrWrongState)
+ }
+ switch c.Op.Phase {
+ case PhaseStarting, PhaseReleasing:
+ if in.Box != c.Op.Intent["source_box"] || in.Coverage == "released" {
+ return fmt.Errorf("%w: only the source admits before pinning", ErrWrongState)
+ }
+ return nil
+ case PhaseProbing:
+ default:
+ return fmt.Errorf("%w: relocating/%s admits no writer", ErrWrongState, c.Op.Phase)
+ }
+ if in.Mode != "cold" || in.Box != c.Op.Exec {
+ return fmt.Errorf("%w: a probe admission is the executor's, cold", ErrWrongState)
+ }
+ p, opened := c.Op.Journal["prepared"], preparedOpened(c)
+ if p == "" || (!opened && in.NewGeneration != p) || (opened && in.NewGeneration == p) {
+ return fmt.Errorf("%w: the first probe opens P, and only it", ErrWrongState)
+ }
+ if !opened {
+ // The opening is journaled with the admission, so it outlives the
+ // admission list's trim (rev 12): P is never opened twice.
+ ensureJournal(c)
+ c.Op.Journal["prepared_opened"] = p
+ }
+ if in.Coverage != "released" {
+ return nil
+ }
+ var r Release
+ var pin RecoveryPin
+ if opened || in.PredecessorCertified || !readJournal(c.Op, "release", &r) || !readJournal(c.Op, "pin", &pin) ||
+ pin.MaxTXID < r.FinalTXID || pin.FilesSeq < r.ManifestSeq {
+ return fmt.Errorf("%w: released coverage needs this operation's release and a pin reaching it", ErrWrongState)
+ }
+ h := c.Holder
+ predecessor := AdmissionRecord{Session: h.Session, Box: h.WriterBox, Generation: h.Generation}
+ if len(h.Admissions) > 0 {
+ predecessor = h.Admissions[len(h.Admissions)-1]
+ }
+ if predecessor.Box != r.ReleasedBy || predecessor.Session != r.Session || predecessor.Generation != r.Generation {
+ return fmt.Errorf("%w: released coverage names another predecessor", ErrWrongState)
+ }
+ return nil
+}
diff --git a/internal/hostclaim/relocation.go b/internal/hostclaim/relocation.go
new file mode 100644
index 000000000..25e89db79
--- /dev/null
+++ b/internal/hostclaim/relocation.go
@@ -0,0 +1,1241 @@
+package hostclaim
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "slices"
+ "strconv"
+ "time"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// Relocation (claim §4 "Same-region relocation", region HA spec 1 §5.8):
+// the reasons, the phases and the bound on target replacements.
+const (
+ ReasonDrain = "drain"
+ ReasonRebalance = "rebalance"
+ ReasonFailover = "failover"
+
+ PhaseStarting = "starting"
+ PhaseReleasing = "releasing"
+ PhasePinning = "pinning"
+ PhaseInstalling = "installing"
+ PhaseInstalled = "installed"
+ PhaseProbing = "probing"
+ PhaseProbeFailed = "probe-failed"
+
+ // MaxTargetReplacements is spec 1 §5.8's bound: after two
+ // replacements of the target in one operation it goes to stuck.
+ MaxTargetReplacements = 2
+
+ // MaxProbeAttempts is §5.8's probe bound: at most three admissions
+ // per target installation within the probe deadline; running out is
+ // probe-failed. Attempts are counted in the claim before they are
+ // made, so a crash or a lost response never buys a fourth.
+ MaxProbeAttempts = 3
+
+ // FailoverPinDelay is §5.8's failover pin rule: pin no earlier than
+ // the fence's observed_at plus 30 seconds, so requests S3 had already
+ // accepted from the terminated box have completed (H3's window).
+ FailoverPinDelay = 30 * time.Second
+)
+
+var (
+ ErrRelocation = errors.New("hostclaim: relocation transition refused")
+ ErrReplacementsExceeded = errors.New("hostclaim: relocation has used its target replacements")
+ ErrAlreadyRecorded = errors.New("hostclaim: relocation journal entry already recorded")
+ ErrProbeAttemptsExhausted = errors.New("hostclaim: the target has used its probe attempts")
+)
+
+// RelocationStart is the start CAS's intent (spec 1 §5.8, claim §4). Fence
+// is set for a failover only and names the source's termination fence.
+type RelocationStart struct {
+ OpID, By string
+ Now time.Time
+ Region, Reason string
+ SourceBox, SourceEnrollment string
+ TargetBox, TargetEnrollment string
+ DirectoryRevision uint64
+ AdmissionID string
+ Fence *FenceRef
+ Discharge *LocalDischarge
+}
+
+// LocalDischarge is a fenced box's volume evidence (claim D5 as amended by
+// L3, spec 1 §5.9, H19): the volume records the caller read and judged with
+// boxvolumes.Discharge. When it is nil, or Qualified is false, the box's
+// park-local chores stay open for the operator path. It never gates the
+// fence or the relocation.
+type LocalDischarge struct {
+ Qualified bool `json:"qualified"`
+ Records []string `json:"records,omitempty"`
+ Reason string `json:"reason,omitempty"`
+}
+
+// Release is the journal entry a live box writes after its release
+// sequence (spec 1 §5.8 steps 1 to 4): its stopped database's final TXID,
+// its final files manifest sequence and its full stop receipt.
+type Release struct {
+ ReleasedBy string `json:"released_by"`
+ Enrollment string `json:"enrollment"`
+ Generation string `json:"generation"`
+ Session uint64 `json:"session"`
+ FinalTXID uint64 `json:"final_txid"`
+ ManifestSeq int64 `json:"manifest_seq"`
+ Receipt string `json:"receipt"`
+ ReceiptSHA256 string `json:"receipt_sha256"`
+}
+
+// ReleaseAuthority verifies a journaled release's full stop receipt: the
+// immutable object at Receipt hashes to ReceiptSHA256 and is complete stop
+// evidence (claim §6.1, P6-D1) for this host, operation, box, incarnation
+// and session. Plan 6b's receipt format is its only implementation.
+type ReleaseAuthority interface {
+ VerifyRelease(context.Context, Claim, Release) error
+}
+
+// RecoveryPin is the journal's reference to one pinned recovery set
+// (§5.8 "The pinned recovery set"): the immutable pin object holding the
+// exact versions, its digest, the database position it reaches, the files
+// manifest it pins, and when it was taken.
+type RecoveryPin struct {
+ Key string `json:"key"`
+ SHA256 string `json:"sha256"`
+ Generation string `json:"generation"`
+ MaxTXID uint64 `json:"max_txid"`
+ FilesManifest string `json:"files_manifest"`
+ FilesSeq int64 `json:"files_seq"`
+ PinnedAt time.Time `json:"pinned_at"`
+ // Empty is the evidenced empty bootstrap (claim §6.4a "Empty
+ // bootstrap", spec 1 §5.8 "A host with nothing replicated"): no LTX
+ // file exists because no transaction was ever committed. It is legal
+ // only when a journaled release reports final_txid 0, or the holder
+ // never admitted a writer (session 0). A listing that merely finds
+ // nothing is never evidence (JournalPin refuses it).
+ Empty bool `json:"empty,omitempty"`
+}
+
+// RecoveryPinKey is where the pin selected against claim generation gen
+// lives: control/relocations/<host>/<op>/pin-<gen>.json. The key binds the
+// object to the exact observation it was selected against, so a pin whose
+// journal CAS lost (the claim moved) is abandoned, never reused, and a
+// retry selects afresh at the new generation (B2-3's round binding).
+func RecoveryPinKey(host, op string, gen uint64) string {
+ return "control/relocations/" + host + "/" + op + "/pin-" + strconv.FormatUint(gen, 10) + ".json"
+}
+
+// PreparedAuthority is the `installed` CAS's record of the remote build
+// (§5.8 "Building the prepared generation"): the install manifest of
+// generation P, verified complete by a fresh read.
+type PreparedAuthority struct {
+ Generation string `json:"generation"`
+ InstallManifest string `json:"install_manifest"`
+ SHA256 string `json:"sha256"`
+ SnapshotTXID uint64 `json:"snapshot_txid"`
+ // FilesSeq and FilesManifestSHA256 identify P's prepared files
+ // manifest (claim rev 6.30 R18-1, moves rev 7.22 §7.6): create-only,
+ // at sequence 0, never replaced; FilesManifestSHA256 is the digest of
+ // the exact bytes a fresh read verified, the exact version the first
+ // admission consumes as its baseline (the session publisher then
+ // starts at sequence 1). SourceFilesSeq is provenance, not identity:
+ // the pinned source manifest's sequence it was built from, which must
+ // be the pin's (rev 15, round 14 finding 1). Required always, an empty
+ // bootstrap included: a P without its files manifest is not built.
+ FilesSeq int64 `json:"files_seq"`
+ SourceFilesSeq int64 `json:"source_files_seq"`
+ FilesManifestSHA256 string `json:"files_manifest_sha256"`
+ // InstallStamp binds the prepared manifest to the installation that
+ // built it ({location, P, op id, installation stamp}): ToInstalled
+ // sets it from install_stamp in the CAS that journals the authority.
+ InstallStamp uint64 `json:"install_stamp,omitempty"`
+}
+
+// AutomaticLoss is claim §7.3 D4's "Automatic loss persistence": what the
+// transition that consumed a termination fence retains on the claim itself,
+// outside the replaceable operation, until the console materializes the
+// D4 host loss record. It survives CompleteRelocation, free and holder
+// replacement because it lives beside loss_refs, not in op.journal.
+type AutomaticLoss struct {
+ ID string `json:"id"` // the loss identity, also in loss_refs
+ Box string `json:"box"`
+ Enrollment string `json:"enrollment"`
+ Fence FenceRef `json:"fence"`
+ ObservedAt string `json:"observed_at"`
+ Op string `json:"op"`
+ Kind string `json:"kind"`
+ Phase string `json:"phase"`
+ Role string `json:"role"` // "source" | "target"
+ // Incarnation and Stamp bind the loss to the holder it was taken
+ // against; Generation and Session are the box's last writer session,
+ // whether it is still the writer or was superseded, and LastAdmission
+ // is that session's admission record when the claim holds one (the
+ // consequences the console reports: its coverage and declarations).
+ Incarnation string `json:"incarnation"`
+ Stamp uint64 `json:"stamp"`
+ Generation string `json:"generation,omitempty"`
+ Session uint64 `json:"session,omitempty"`
+ LastAdmission *AdmissionRecord `json:"last_admission,omitempty"`
+ Discharge LocalDischarge `json:"discharge"`
+ // Reassigned is the regional work (park-generation, park-replica)
+ // the box owned that the same CAS passed to its region (L3, as
+ // DeclareLost's reassign_regional): the console's D4 record names it.
+ Reassigned []ObligationRef `json:"reassigned,omitempty"`
+ Timers TimerKnowledge `json:"timers"`
+ // Reported is set by the console once the D4 record holds this loss
+ // and its alert work exists. Until then reporting is pending.
+ Reported bool `json:"reported,omitempty"`
+}
+
+// automaticLossID is deterministic for one fence consumed by one
+// operation for one box, so a replayed transition names the same loss and
+// the console's materialization deduplicates on it (D4: "Retries
+// deduplicate by the retained loss identity").
+func automaticLossID(opID, box string, ref FenceRef) string {
+ sum := sha256.Sum256([]byte(opID + "\x00" + box + "\x00" + ref.Enrollment + "\x00" + ref.SHA256))
+ return "auto-" + hex.EncodeToString(sum[:16])
+}
+
+// LocalInstall is one box's local install receipt (§5.8, B1-2, B2-5).
+type LocalInstall struct {
+ Box string `json:"box"`
+ Enrollment string `json:"enrollment"`
+ InstallStamp uint64 `json:"install_stamp"`
+ FilesManifestVersion int64 `json:"files_manifest_version"`
+ JournalSHA256 string `json:"journal_sha256"`
+}
+
+// Probe is the target's journaled probe result for its admitted session.
+type Probe struct {
+ Box string `json:"box"`
+ Session uint64 `json:"session"`
+}
+
+// Target names a box for the relocation's target slot.
+type Target struct {
+ Box, Enrollment string
+}
+
+func journalJSON(c *Claim, key string, v any) error {
+ raw, err := json.Marshal(v)
+ if err != nil {
+ return err
+ }
+ if c.Op.Journal == nil {
+ c.Op.Journal = map[string]string{}
+ }
+ c.Op.Journal[key] = string(raw)
+ return nil
+}
+
+func readJournal(op Op, key string, v any) bool {
+ raw, ok := op.Journal[key]
+ return ok && json.Unmarshal([]byte(raw), v) == nil
+}
+
+func journalUint(op Op, key string) uint64 {
+ n, _ := strconv.ParseUint(op.Journal[key], 10, 64)
+ return n
+}
+
+func requireRelocation(c *Claim, opID string, phases ...string) error {
+ if err := RequireOp(opID)(c); err != nil {
+ return err
+ }
+ if c.Holder == nil || c.Op.Kind != KindRelocating || !slices.Contains(phases, c.Op.Phase) {
+ return fmt.Errorf("%w: %s/%s", ErrWrongState, c.Op.Kind, c.Op.Phase)
+ }
+ if c.Op.Journal["stuck"] != "" {
+ return fmt.Errorf("%w: operation is stuck", ErrRelocation)
+ }
+ ensureJournal(c)
+ return nil
+}
+
+// ensureJournal makes the journal writable: an empty one round-trips as nil.
+func ensureJournal(c *Claim) {
+ if c.Op.Journal == nil {
+ c.Op.Journal = map[string]string{}
+ }
+}
+
+// requireAcks is the D6 barrier: every current participant has recorded
+// its execution acknowledgement before the operation leaves its phase.
+func requireAcks(c *Claim) error {
+ for _, box := range c.Op.Participants {
+ if c.Op.Journal["exec_ack."+box] == "" {
+ return fmt.Errorf("%w: %s has not acknowledged", ErrRelocation, box)
+ }
+ }
+ return nil
+}
+
+func failover(op Op) bool {
+ return op.Intent["reason"] == ReasonFailover || op.Journal["reason"] == ReasonFailover
+}
+
+// RelocationStartGuard is §4's start guard with the relocation exception
+// (L11): a park-generation chore in the region does not block, a
+// park-replica in the region does, a park-local on the target does, and a
+// park-local on a fenced source does not block a failover.
+func RelocationStartGuard(c *Claim, region, source, target string, sourceFenced bool) error {
+ if c.Holder == nil || c.Op.Kind != KindServing || c.Op.ID != "" {
+ return fmt.Errorf("%w: kind %q", ErrWrongState, c.Op.Kind)
+ }
+ for _, o := range c.Obligations {
+ switch {
+ case o.Kind == "park-replica" && o.Region == region:
+ return fmt.Errorf("%w: park-replica in %s", ErrPendingChore, region)
+ case o.Kind == "park-local" && o.Box == target:
+ return fmt.Errorf("%w: park-local on %s", ErrPendingChore, target)
+ case o.Kind == "park-local" && o.Box == source && !sourceFenced:
+ return fmt.Errorf("%w: park-local on %s", ErrPendingChore, source)
+ }
+ }
+ return nil
+}
+
+// OwesLocal reports whether box owes a park-local chore on this claim: a
+// local retirement not yet done (spec 1 §2: such a box is never chosen to
+// install the host, and §4: a park-local on the target refuses).
+func OwesLocal(c *Claim, box string) bool {
+ for _, o := range c.Obligations {
+ if o.Kind == "park-local" && o.Box == box {
+ return true
+ }
+ }
+ return false
+}
+
+// LocalChoreBoxes lists every box that owes a park-local chore on the
+// claim, for a chooser's exclusions.
+func LocalChoreBoxes(c *Claim) []string {
+ var out []string
+ for _, o := range c.Obligations {
+ if o.Kind == "park-local" && o.Box != "" && !slices.Contains(out, o.Box) {
+ out = append(out, o.Box)
+ }
+ }
+ return out
+}
+
+// fenceBox puts box on lost with its fence reference (L2, L4), passes the
+// regional work the box owned (park-generation, park-replica owned by
+// box:<box>) to its region, region:<r>, keeping each obligation's id,
+// operation, retirement, region, generation and park scope (L3, exactly
+// as DeclareLost does for a declared loss, lost.go), and, when the volume
+// evidence qualifies (L3), discharges that box's park-local chores for
+// this host, all in the same CAS. It never removes anything else. D4: the
+// same CAS retains the loss outside the operation (evidence_refs,
+// loss_refs) with the box's last writer session, the discharge verdict and
+// timer knowledge "unknown", so the console can materialize the loss record
+// after the operation is long gone.
+func fenceBox(c *Claim, box string, ref FenceRef, observedAt string, d *LocalDischarge) error {
+ key := "fence." + box
+ raw, err := json.Marshal(ref)
+ if err != nil {
+ return err
+ }
+ if old, ok := c.Op.Journal[key]; ok && old != string(raw) {
+ return fmt.Errorf("%w: fence reference already bound", ErrFence)
+ }
+ if c.Op.Journal == nil {
+ c.Op.Journal = map[string]string{}
+ }
+ c.Op.Journal[key] = string(raw)
+ c.Op.Journal[key+".observed_at"] = observedAt
+ if !slices.Contains(c.Lost, box) {
+ c.Lost = append(c.Lost, box)
+ }
+ var reassigned []ObligationRef
+ for i := range c.Obligations {
+ o := &c.Obligations[i]
+ if (o.Kind == "park-generation" || o.Kind == "park-replica") && o.Owner == "box:"+box {
+ if o.Region == "" {
+ return fmt.Errorf("%w: regional obligation %q names no region", ErrWrongState, o.ID)
+ }
+ o.Owner = "region:" + o.Region
+ reassigned = append(reassigned, ObligationRef{ID: o.ID, Op: o.Op, Retirement: o.Retirement})
+ }
+ }
+ if d == nil {
+ d = &LocalDischarge{Reason: "no volume evidence supplied"}
+ }
+ if err := journalJSON(c, "discharge."+box, d); err != nil {
+ return err
+ }
+ if d.Qualified {
+ // Each chore is completed with CompleteObligation's own semantics,
+ // so a discovery operation whose final chore this discharges is
+ // marked complete exactly as a retirement would (delete.go).
+ for {
+ var chore *Obligation
+ for i := range c.Obligations {
+ if c.Obligations[i].Kind == "park-local" && c.Obligations[i].Box == box {
+ chore = &c.Obligations[i]
+ break
+ }
+ }
+ if chore == nil {
+ break
+ }
+ if err := CompleteObligation(chore.Op, chore.ID)(c); err != nil {
+ return err
+ }
+ }
+ }
+ return retainAutomaticLoss(c, box, ref, observedAt, *d, reassigned)
+}
+
+// retainAutomaticLoss is D4's retention, written once per (operation, box,
+// fence): a replay finds its record and changes nothing.
+func retainAutomaticLoss(c *Claim, box string, ref FenceRef, observedAt string, d LocalDischarge, reassigned []ObligationRef) error {
+ id := automaticLossID(c.Op.ID, box, ref)
+ for _, l := range c.EvidenceRefs {
+ if l.ID == id {
+ return nil
+ }
+ }
+ role := "target"
+ if box == c.Op.Intent["source_box"] {
+ role = "source"
+ }
+ loss := AutomaticLoss{ID: id, Box: box, Enrollment: ref.Enrollment, Fence: ref, ObservedAt: observedAt,
+ Op: c.Op.ID, Kind: c.Op.Kind, Phase: c.Op.Phase, Role: role, Discharge: d, Reassigned: reassigned, Timers: TimersUnknown}
+ if h := c.Holder; h != nil {
+ loss.Incarnation, loss.Stamp = h.Incarnation, h.Stamp
+ // The box's last writer session: the live one when it is still
+ // the writer, otherwise its last admission on this holder.
+ if h.WriterBox == box {
+ loss.Generation, loss.Session = h.Generation, h.Session
+ }
+ for i := len(h.Admissions) - 1; i >= 0; i-- {
+ if h.Admissions[i].Box == box {
+ last := h.Admissions[i]
+ loss.LastAdmission = &last
+ if loss.Session == 0 {
+ loss.Generation, loss.Session = last.Generation, last.Session
+ }
+ break
+ }
+ }
+ }
+ c.EvidenceRefs = append(c.EvidenceRefs, loss)
+ if !slices.Contains(c.LossRefs, id) {
+ c.LossRefs = append(c.LossRefs, id)
+ }
+ return nil
+}
+
+func removeParticipant(c *Claim, box string) {
+ c.Op.Participants = slices.DeleteFunc(slices.Clone(c.Op.Participants), func(b string) bool { return b == box })
+}
+
+func addParticipant(c *Claim, box string) {
+ if !slices.Contains(c.Op.Participants, box) {
+ c.Op.Participants = append(c.Op.Participants, box)
+ }
+}
+
+// installedBoxes is every box this operation handed op.exec for an
+// installation; each gets a park-local unless discharged (claim §4:
+// "Every abandoned local installation receives an exact park-local").
+func noteInstalledBox(c *Claim, box string) error {
+ var boxes []string
+ readJournal(c.Op, "installed_boxes", &boxes)
+ if !slices.Contains(boxes, box) {
+ boxes = append(boxes, box)
+ }
+ return journalJSON(c, "installed_boxes", boxes)
+}
+
+// StartRelocation is the start CAS (spec 1 §5.8, claim §4). A drain or
+// rebalance starts with op.exec on the source; a failover cites the
+// source's termination fence, starts with op.exec "none", keeps the stamp,
+// and puts the source on lost. The caller has already checked the roster
+// (hostactors.Directory.CheckBoxStart), the target's eligibility and the
+// app-admission registration; this CAS records what it checked.
+func StartRelocation(ctx context.Context, s release.Store, observed Observed, in RelocationStart, authority FenceAuthority) (Observed, error) {
+ if observed.Absent || observed.Claim == nil || observed.ETag == "" {
+ return Observed{}, ErrWrongState
+ }
+ switch in.Reason {
+ case ReasonDrain, ReasonRebalance:
+ if in.Fence != nil {
+ return Observed{}, fmt.Errorf("%w: only a failover cites a fence", ErrRelocation)
+ }
+ case ReasonFailover:
+ if in.Fence == nil {
+ return Observed{}, fmt.Errorf("%w: a failover cites the source's fence", ErrRelocation)
+ }
+ default:
+ return Observed{}, fmt.Errorf("%w: reason %q", ErrRelocation, in.Reason)
+ }
+ if in.OpID == "" || in.Region == "" || in.AdmissionID == "" || in.DirectoryRevision == 0 || in.Now.IsZero() ||
+ !validBoxReference(in.SourceBox) || !validBoxReference(in.TargetBox) || in.SourceBox == "none" || in.TargetBox == "none" ||
+ in.SourceBox == in.TargetBox || in.SourceEnrollment == "" || in.TargetEnrollment == "" {
+ return Observed{}, fmt.Errorf("%w: incomplete start", ErrRelocation)
+ }
+ var observedAt string
+ if in.Fence != nil {
+ v, err := VerifyFence(ctx, s, authority, *in.Fence)
+ if err != nil {
+ return Observed{}, err
+ }
+ if v.record.Box != in.SourceBox || v.record.Enrollment != in.SourceEnrollment || v.binding.Region != in.Region {
+ return Observed{}, ErrFence
+ }
+ observedAt = v.record.ObservedAt
+ }
+ return Update(ctx, s, observed, func(c *Claim) error {
+ if c.Op.Exec != in.SourceBox || slices.Contains(c.Lost, in.TargetBox) {
+ return fmt.Errorf("%w: the source is not the executor, or the target is lost", ErrWrongState)
+ }
+ if err := RelocationStartGuard(c, in.Region, in.SourceBox, in.TargetBox, in.Fence != nil); err != nil {
+ return err
+ }
+ exec, participants := in.SourceBox, []string{in.SourceBox, in.TargetBox}
+ if in.Fence != nil {
+ exec, participants = "none", []string{in.TargetBox}
+ }
+ c.Op = Op{Kind: KindRelocating, ID: in.OpID, Phase: PhaseStarting, By: in.By, StartedAt: in.Now.UTC(),
+ Exec: exec, Participants: participants,
+ Intent: map[string]string{
+ "region": in.Region, "reason": in.Reason,
+ "source_box": in.SourceBox, "source_enrollment": in.SourceEnrollment,
+ "target_box": in.TargetBox, "target_enrollment": in.TargetEnrollment,
+ "stamp_from": strconv.FormatUint(c.Holder.Stamp, 10),
+ "generation_from": c.Holder.Generation,
+ "directory_revision": strconv.FormatUint(in.DirectoryRevision, 10),
+ "admission_id": in.AdmissionID,
+ },
+ Journal: map[string]string{}}
+ if in.Fence != nil {
+ return fenceBox(c, in.SourceBox, *in.Fence, observedAt, in.Discharge)
+ }
+ return nil
+ })
+}
+
+// FenceSource is §5.8's source-loss rule (B1-6): a drain or rebalance
+// whose source is terminated becomes a failover in one CAS citing the
+// source's fence. In starting it stays in starting with the source off the
+// barrier; in releasing before a release is journaled it goes to pinning
+// with the fence as evidence; after a release (pinning on) the source only
+// goes on lost. Every case keeps the stamp and the intent.
+func FenceSource(ctx context.Context, s release.Store, observed Observed, opID string, ref FenceRef, d *LocalDischarge, authority FenceAuthority) (Observed, error) {
+ if observed.Absent || observed.Claim == nil {
+ return Observed{}, ErrWrongState
+ }
+ before := observed.Claim.Op
+ source := before.Intent["source_box"]
+ v, err := VerifyFence(ctx, s, authority, ref)
+ if err != nil {
+ return Observed{}, err
+ }
+ if v.record.Box != source || v.record.Enrollment != before.Intent["source_enrollment"] || v.binding.Region != before.Intent["region"] {
+ return Observed{}, ErrFence
+ }
+ return Update(ctx, s, observed, func(c *Claim) error {
+ if err := RequireOp(opID)(c); err != nil {
+ return err
+ }
+ if c.Op.Kind != KindRelocating || c.Op.Intent["source_box"] != source {
+ return ErrWrongState
+ }
+ ensureJournal(c)
+ // Before its release is journaled the source's writes are not
+ // kept by a release, so the op becomes a failover. After it, only
+ // the source's liveness changes.
+ if c.Op.Phase == PhaseStarting || (c.Op.Phase == PhaseReleasing && c.Op.Journal["release"] == "") {
+ c.Op.Journal["reason"] = ReasonFailover
+ removeParticipant(c, source)
+ c.Op.Exec = "none"
+ if c.Op.Phase == PhaseReleasing {
+ c.Op.Phase = PhasePinning
+ }
+ }
+ return fenceBox(c, source, ref, v.record.ObservedAt, d)
+ })
+}
+
+// ToReleasing is a drain's barrier CAS: from starting, once every
+// participant has acknowledged. A failover never enters releasing. now
+// starts the release deadline (hostrelocate.ReleaseDeadline): a source
+// that never journals its release goes to stuck.
+func ToReleasing(opID string, now time.Time) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := requireRelocation(c, opID, PhaseStarting); err != nil {
+ return err
+ }
+ if failover(c.Op) {
+ return fmt.Errorf("%w: a failover skips releasing", ErrRelocation)
+ }
+ if err := requireAcks(c); err != nil {
+ return err
+ }
+ if now.IsZero() {
+ return fmt.Errorf("%w: releasing needs its start time", ErrRelocation)
+ }
+ c.Op.Phase = PhaseReleasing
+ c.Op.Journal["releasing_since"] = now.UTC().Format(time.RFC3339Nano)
+ return nil
+ }
+}
+
+// JournalRelease records a live box's completed release sequence. The
+// source records it in releasing; a failed target in probe-failed. The
+// release must be of the box's own current writer session.
+func JournalRelease(opID string, r Release) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := requireRelocation(c, opID, PhaseReleasing, PhaseProbeFailed); err != nil {
+ return err
+ }
+ h := c.Holder
+ writerMismatch := h.Session != 0 && h.WriterBox != r.ReleasedBy // a host never woken has no writer
+ if r.ReleasedBy == "" || r.ReleasedBy != c.Op.Exec || r.Enrollment == "" || r.Receipt == "" || !isHex64(r.ReceiptSHA256) ||
+ r.Generation != h.Generation || r.Session != h.Session || writerMismatch {
+ return fmt.Errorf("%w: release is not the executor's current session", ErrRelocation)
+ }
+ want := c.Op.Intent["source_enrollment"]
+ if c.Op.Phase == PhaseProbeFailed {
+ want = c.Op.Intent["target_enrollment"]
+ }
+ if r.Enrollment != want {
+ return fmt.Errorf("%w: release enrollment", ErrRelocation)
+ }
+ var old Release
+ if readJournal(c.Op, "release", &old) && c.Op.Journal["release.phase"] == c.Op.Phase {
+ if old == r {
+ return ErrAlreadyRecorded
+ }
+ return fmt.Errorf("%w: a different release is journaled", ErrRelocation)
+ }
+ c.Op.Journal["release.phase"] = c.Op.Phase
+ return journalJSON(c, "release", r)
+ }
+}
+
+// PinAfterRelease leaves releasing (a drain) or probe-failed (a failed
+// target, §5.8's nested relocation) for pinning, citing the journaled
+// release's verified receipt. op.exec becomes "none" and the stamp is kept
+// (claim R6-4, D5's non-installing form). From probe-failed, next names
+// the replacement target, which must not be the failed one.
+func PinAfterRelease(ctx context.Context, s release.Store, observed Observed, opID string, authority ReleaseAuthority, next *Target) (Observed, error) {
+ if observed.Absent || observed.Claim == nil || authority == nil {
+ return Observed{}, ErrWrongState
+ }
+ var r Release
+ if !readJournal(observed.Claim.Op, "release", &r) {
+ return Observed{}, fmt.Errorf("%w: no journaled release", ErrRelocation)
+ }
+ if err := authority.VerifyRelease(ctx, *observed.Claim, r); err != nil {
+ return Observed{}, err
+ }
+ return Update(ctx, s, observed, func(c *Claim) error {
+ if err := requireRelocation(c, opID, PhaseReleasing, PhaseProbeFailed); err != nil {
+ return err
+ }
+ var now Release
+ if !readJournal(c.Op, "release", &now) || now != r || c.Op.Journal["release.phase"] != c.Op.Phase || r.ReleasedBy != c.Op.Exec {
+ return fmt.Errorf("%w: release changed", ErrRelocation)
+ }
+ if c.Op.Phase == PhaseReleasing {
+ if next != nil {
+ return fmt.Errorf("%w: a drain keeps its target", ErrRelocation)
+ }
+ // §5.8: a target replaced in releasing acknowledges before the
+ // operation leaves the phase.
+ if err := requireAcks(c); err != nil {
+ return err
+ }
+ } else {
+ if next == nil || next.Box == r.ReleasedBy {
+ return fmt.Errorf("%w: a nested relocation names another target", ErrRelocation)
+ }
+ if err := replaceTargetSlot(c, *next); err != nil {
+ return err
+ }
+ c.Op.Journal["released_by_target"] = r.ReleasedBy
+ }
+ c.Op.Exec, c.Op.Phase = "none", PhasePinning
+ delete(c.Op.Journal, "pin")
+ return nil
+ })
+}
+
+// PinFailover leaves starting for pinning on a failover: the source's
+// fence is journaled and every remaining participant has acknowledged.
+func PinFailover(opID string) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := requireRelocation(c, opID, PhaseStarting); err != nil {
+ return err
+ }
+ if !failover(c.Op) || c.Op.Journal["fence."+c.Op.Intent["source_box"]] == "" || c.Op.Exec != "none" {
+ return fmt.Errorf("%w: pinning without a release needs the source's fence", ErrRelocation)
+ }
+ if err := requireAcks(c); err != nil {
+ return err
+ }
+ c.Op.Phase = PhasePinning
+ return nil
+ }
+}
+
+// JournalPin records the pinned recovery set in pinning. The pin's key
+// must be RecoveryPinKey for this claim generation: the object was selected
+// against exactly this observation (B2-3). After a release it must reach
+// the journaled final TXID and files sequence; on a failover it must be
+// taken no earlier than the fence's observed_at plus FailoverPinDelay. An
+// Empty pin needs its evidence: a release at final_txid 0, or a holder
+// that never admitted a writer.
+func JournalPin(opID string, p RecoveryPin) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := requireRelocation(c, opID, PhasePinning); err != nil {
+ return err
+ }
+ if p.Key != RecoveryPinKey(c.Host, opID, c.Gen) {
+ return fmt.Errorf("%w: pin %s was not selected against this observation (gen %d)", ErrRelocation, p.Key, c.Gen)
+ }
+ if !isHex64(p.SHA256) || p.FilesManifest == "" || p.PinnedAt.IsZero() || (p.Empty != (p.MaxTXID == 0)) {
+ return fmt.Errorf("%w: incomplete pin", ErrRelocation)
+ }
+ var r Release
+ released := readJournal(c.Op, "release", &r)
+ if released {
+ if p.MaxTXID < r.FinalTXID || p.FilesSeq < r.ManifestSeq {
+ return fmt.Errorf("%w: pin %d/%d is short of the release's %d/%d", ErrRelocation, p.MaxTXID, p.FilesSeq, r.FinalTXID, r.ManifestSeq)
+ }
+ }
+ if p.Empty && !((released && r.FinalTXID == 0) || (!released && c.Holder.Session == 0)) {
+ return fmt.Errorf("%w: an empty pin needs a release at txid 0 or a holder that never had a writer", ErrRelocation)
+ }
+ for key, at := range c.Op.Journal {
+ if len(key) > len(".observed_at") && key[len(key)-len(".observed_at"):] == ".observed_at" {
+ t, err := time.Parse(time.RFC3339Nano, at)
+ if err != nil || p.PinnedAt.Before(t.Add(FailoverPinDelay)) {
+ return fmt.Errorf("%w: pinned before the fence's observation plus %s", ErrRelocation, FailoverPinDelay)
+ }
+ }
+ }
+ if readJournal(c.Op, "pin", new(RecoveryPin)) {
+ return ErrAlreadyRecorded
+ }
+ return journalJSON(c, "pin", p)
+ }
+}
+
+// NameTarget names the target of an operation that has none: a move
+// converted to relocating before its flip (claim §4, moves §7.10). The
+// named box acknowledges (D6) before ToInstalling hands it op.exec.
+func NameTarget(opID string, t Target) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := requireRelocation(c, opID, PhasePinning); err != nil {
+ return err
+ }
+ if c.Op.Intent["target_box"] != "" {
+ return fmt.Errorf("%w: the operation already has a target", ErrRelocation)
+ }
+ return replaceTargetSlot(c, t)
+ }
+}
+
+// ToInstalling hands op.exec to the target for a new installation: a new
+// stamp (this CAS's gen, R6-4), journaled as install_stamp, a fresh
+// prepared generation id, and the target added to touched.boxes before its
+// first effect. A converted operation with no target names it here.
+func ToInstalling(opID string, t Target, prepared string) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := requireRelocation(c, opID, PhasePinning); err != nil {
+ return err
+ }
+ if !readJournal(c.Op, "pin", new(RecoveryPin)) || prepared == "" || prepared == c.Holder.Generation {
+ return fmt.Errorf("%w: installing needs a pin and a fresh prepared generation", ErrRelocation)
+ }
+ if t.Box != c.Op.Intent["target_box"] || t.Enrollment != c.Op.Intent["target_enrollment"] || slices.Contains(c.Lost, t.Box) {
+ return fmt.Errorf("%w: not the operation's target", ErrRelocation)
+ }
+ if OwesLocal(c, t.Box) {
+ return fmt.Errorf("%w: park-local on %s", ErrPendingChore, t.Box)
+ }
+ if err := requireAcks(c); err != nil {
+ return err
+ }
+ return install(c, t.Box, prepared, PhaseInstalling)
+ }
+}
+
+// abandonPrepared journals the operation's current P as abandoned when
+// no writer opened it, before a transition replaces or drops it: nothing
+// will open it now, and the serving CAS parks it (rev 17, finding F3;
+// rev 18: every transition that discards an unopened P calls it).
+func abandonPrepared(c *Claim) error {
+ old := c.Op.Journal["prepared"]
+ if old == "" || preparedOpened(c) {
+ return nil
+ }
+ var abandoned []string
+ readJournal(c.Op, "abandoned_prepared", &abandoned)
+ if slices.Contains(abandoned, old) {
+ return nil
+ }
+ return journalJSON(c, "abandoned_prepared", append(abandoned, old))
+}
+
+func install(c *Claim, box, prepared, phase string) error {
+ if c.Op.Journal["prepared"] != prepared {
+ if err := abandonPrepared(c); err != nil {
+ return err
+ }
+ }
+ stamp := c.Gen + 1 // the resulting gen: Update sets gen = observed+1
+ c.Op.Exec, c.Op.Phase = box, phase
+ c.Holder.Stamp = stamp
+ c.Op.Journal["install_stamp"] = strconv.FormatUint(stamp, 10)
+ c.Op.Journal["prepared"] = prepared
+ delete(c.Op.Journal, "local_install")
+ if !slices.Contains(c.Touched.Boxes, box) {
+ c.Touched.Boxes = append(c.Touched.Boxes, box)
+ }
+ return noteInstalledBox(c, box)
+}
+
+// ToInstalled records the verified remote build of P (§5.8, the
+// no-reinstall boundary): the install manifest at the pin's N (none for an
+// empty pin) and, always, P's files manifest at the pinned sequence, each
+// verified by a fresh read. Nobody builds P again after this CAS.
+func ToInstalled(opID string, a PreparedAuthority) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := requireRelocation(c, opID, PhaseInstalling); err != nil {
+ return err
+ }
+ var pin RecoveryPin
+ readJournal(c.Op, "pin", &pin)
+ if a.Generation == "" || a.Generation != c.Op.Journal["prepared"] || a.SnapshotTXID != pin.MaxTXID ||
+ (pin.MaxTXID > 0 && (a.InstallManifest == "" || !isHex64(a.SHA256))) ||
+ (pin.MaxTXID == 0 && (a.InstallManifest != "" || a.SHA256 != "")) {
+ return fmt.Errorf("%w: prepared authority is not this pin's build of P", ErrRelocation)
+ }
+ if a.FilesSeq != 0 || a.SourceFilesSeq != pin.FilesSeq || !isHex64(a.FilesManifestSHA256) {
+ return fmt.Errorf("%w: prepared authority lacks P's prepared files manifest (sequence 0, built from the pinned sequence)", ErrRelocation)
+ }
+ if err := requireAcks(c); err != nil {
+ return err
+ }
+ c.Op.Phase = PhaseInstalled
+ a.InstallStamp = journalUint(c.Op, "install_stamp")
+ return journalJSON(c, "prepared_authority", a)
+ }
+}
+
+// PreparedOpened reports whether P has been opened: journaled as
+// prepared_opened by the admission that opened it (rev 12: AdmitWriter
+// trims holder.admissions to eight, so the admission list alone could
+// forget it), or named by a retained admission.
+func PreparedOpened(c *Claim) bool { return preparedOpened(c) }
+
+func preparedOpened(c *Claim) bool {
+ p := c.Op.Journal["prepared"]
+ if p == "" || c.Holder == nil {
+ return false
+ }
+ if c.Op.Journal["prepared_opened"] == p {
+ return true
+ }
+ for _, a := range c.Holder.Admissions {
+ if a.Generation == p {
+ return true
+ }
+ }
+ return false
+}
+
+// RecordLocalInstall journals the executor's local install receipt in
+// installed (B1-2). It is bound to op.exec, the target's enrollment and the
+// journaled install_stamp, and refused once any admission has named P
+// (B2-5): after that the local install is never remade.
+func RecordLocalInstall(opID string, l LocalInstall) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := requireRelocation(c, opID, PhaseInstalled); err != nil {
+ return err
+ }
+ if l.Box != c.Op.Exec || l.Enrollment != c.Op.Intent["target_enrollment"] ||
+ l.InstallStamp == 0 || l.InstallStamp != journalUint(c.Op, "install_stamp") || !isHex64(l.JournalSHA256) {
+ return fmt.Errorf("%w: receipt is not this installation's", ErrRelocation)
+ }
+ if preparedOpened(c) {
+ return fmt.Errorf("%w: P was admitted; the local install is never remade", ErrRelocation)
+ }
+ var old LocalInstall
+ if readJournal(c.Op, "local_install", &old) {
+ if old == l {
+ return ErrAlreadyRecorded
+ }
+ return fmt.Errorf("%w: a different receipt is journaled", ErrRelocation)
+ }
+ return journalJSON(c, "local_install", l)
+ }
+}
+
+// ToProbing enters probing, which requires a local install receipt for
+// exactly op.exec, its enrollment and the journaled install_stamp. now
+// starts the probe deadline (§5.8: 10 minutes).
+func ToProbing(opID string, now time.Time) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := requireRelocation(c, opID, PhaseInstalled); err != nil {
+ return err
+ }
+ var l LocalInstall
+ if !readJournal(c.Op, "local_install", &l) || l.Box != c.Op.Exec || l.Enrollment != c.Op.Intent["target_enrollment"] ||
+ l.InstallStamp != journalUint(c.Op, "install_stamp") {
+ return fmt.Errorf("%w: no local install receipt for the executor", ErrRelocation)
+ }
+ if err := requireAcks(c); err != nil {
+ return err
+ }
+ if now.IsZero() {
+ return fmt.Errorf("%w: probing needs its start time", ErrRelocation)
+ }
+ c.Op.Phase = PhaseProbing
+ c.Op.Journal["probing_since"] = now.UTC().Format(time.RFC3339Nano)
+ return nil
+ }
+}
+
+// BeginProbeAttempt counts one probe attempt for the executor before it
+// is made (§5.8 "limits"): the fourth is refused with
+// ErrProbeAttemptsExhausted, and the participant then enters
+// probe-failed. The count is per box and per operation, so a replacement
+// target starts at zero and a fenced one is never counted again.
+func BeginProbeAttempt(opID, box string) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := requireRelocation(c, opID, PhaseProbing); err != nil {
+ return err
+ }
+ if box == "" || box != c.Op.Exec {
+ return fmt.Errorf("%w: only the executor probes", ErrRelocation)
+ }
+ key := "probe_attempts." + box
+ n := journalUint(c.Op, key)
+ if n >= MaxProbeAttempts {
+ return ErrProbeAttemptsExhausted
+ }
+ c.Op.Journal[key] = strconv.FormatUint(n+1, 10)
+ return nil
+ }
+}
+
+// RecordProbe journals the executor's passing probe of its admitted session.
+func RecordProbe(opID string, p Probe) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := requireRelocation(c, opID, PhaseProbing); err != nil {
+ return err
+ }
+ if p.Box != c.Op.Exec || p.Box != c.Holder.WriterBox || p.Session == 0 || p.Session != c.Holder.Session ||
+ p.Session < journalUint(c.Op, "install_stamp") {
+ return fmt.Errorf("%w: probe is not of the executor's admitted session", ErrRelocation)
+ }
+ return journalJSON(c, "probe", p)
+ }
+}
+
+// ToProbeFailed records that the live target's probe failed or missed its
+// deadline. The target then runs its release sequence (JournalRelease);
+// now starts its release deadline.
+func ToProbeFailed(opID string, now time.Time) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := requireRelocation(c, opID, PhaseProbing); err != nil {
+ return err
+ }
+ if now.IsZero() {
+ return fmt.Errorf("%w: probe-failed needs its start time", ErrRelocation)
+ }
+ c.Op.Phase = PhaseProbeFailed
+ c.Op.Journal["probe_failed_since"] = now.UTC().Format(time.RFC3339Nano)
+ delete(c.Op.Journal, "release")
+ delete(c.Op.Journal, "release.phase")
+ return nil
+ }
+}
+
+// CompleteRelocation is the serving CAS: the target's probe passed for its
+// admitted session, so the host serves on it alone. Every other box this
+// operation installed on, and a live source, owes a park-local unless a
+// fence discharged it.
+func CompleteRelocation(opID string, newID func() string) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := requireRelocation(c, opID, PhaseProbing); err != nil {
+ return err
+ }
+ var p Probe
+ if !readJournal(c.Op, "probe", &p) || p.Box != c.Op.Exec || p.Session != c.Holder.Session || c.Holder.WriterBox != p.Box {
+ return fmt.Errorf("%w: no passing probe of the current session", ErrRelocation)
+ }
+ target := c.Op.Exec
+ var owed []string
+ readJournal(c.Op, "installed_boxes", &owed)
+ owed = append([]string{c.Op.Intent["source_box"]}, owed...)
+ for _, box := range owed {
+ if box == target || box == "" {
+ continue
+ }
+ var d LocalDischarge
+ if readJournal(c.Op, "discharge."+box, &d) && d.Qualified {
+ continue
+ }
+ exists := false
+ for _, o := range c.Obligations {
+ if o.Kind == "park-local" && o.Box == box {
+ exists = true
+ }
+ }
+ if !exists {
+ c.Obligations = append(c.Obligations, Obligation{ID: newID(), Op: opID, Kind: "park-local", Box: box, Retirement: newID(), Owner: "box:" + box})
+ }
+ }
+ // Every prepared generation this operation built and no writer
+ // opened is parked here, in the region, by the claim's own
+ // park-generation chore (rev 17, finding F3); the at-rest sweep
+ // stays the backstop the spec names.
+ var abandoned []string
+ readJournal(c.Op, "abandoned_prepared", &abandoned)
+ region := c.Op.Intent["region"]
+ for _, g := range abandoned {
+ if g == "" || g == c.Holder.Generation || region == "" {
+ continue
+ }
+ queued := false
+ for _, o := range c.Obligations {
+ if o.Kind == "park-generation" && o.Region == region && o.Generation == g {
+ queued = true
+ }
+ }
+ if !queued {
+ c.Obligations = append(c.Obligations, Obligation{ID: newID(), Op: opID, Kind: "park-generation", Region: region, Generation: g, Retirement: newID(), Owner: "region:" + region})
+ }
+ }
+ c.Op = Op{Kind: KindServing, Exec: target, Participants: []string{target}}
+ return nil
+ }
+}
+
+// replaceTargetSlot names t as the target, counting a replacement when it
+// replaces another box. A replaced box is never named again for this host.
+func replaceTargetSlot(c *Claim, t Target) error {
+ if !validBoxReference(t.Box) || t.Box == "none" || t.Enrollment == "" || t.Box == c.Op.Intent["source_box"] || slices.Contains(c.Lost, t.Box) {
+ return fmt.Errorf("%w: invalid target", ErrRelocation)
+ }
+ var excluded []string
+ readJournal(c.Op, "excluded_targets", &excluded)
+ if slices.Contains(excluded, t.Box) {
+ return fmt.Errorf("%w: %s was replaced in this operation", ErrRelocation, t.Box)
+ }
+ if OwesLocal(c, t.Box) {
+ // Spec 1 §2: a box owing a local retirement for this host is never
+ // handed an installation, in any phase a replacement resumes at
+ // (install is called below without ToInstalling's check).
+ return fmt.Errorf("%w: park-local on %s", ErrPendingChore, t.Box)
+ }
+ old := c.Op.Intent["target_box"]
+ if old != "" && old != t.Box {
+ n := journalUint(c.Op, "replacements")
+ if n >= MaxTargetReplacements {
+ return ErrReplacementsExceeded
+ }
+ c.Op.Journal["replacements"] = strconv.FormatUint(n+1, 10)
+ excluded = append(excluded, old)
+ if err := journalJSON(c, "excluded_targets", excluded); err != nil {
+ return err
+ }
+ removeParticipant(c, old)
+ }
+ c.Op.Intent["target_box"], c.Op.Intent["target_enrollment"] = t.Box, t.Enrollment
+ addParticipant(c, t.Box)
+ return nil
+}
+
+// ReplaceTarget is §5.8's replacement table for a target terminated by its
+// fence. It records the fence (the old target goes on lost), names next,
+// and resumes at the phase the table gives:
+//
+// starting, releasing, pinning -> the same phase
+// installing -> installing: next is op.exec, new stamp, fresh P'
+// installed, or probing before P -> installed: next is op.exec, new stamp, same P, no receipt
+// probing after P, or probe-failed -> pinning: a nested failover from holder.generation
+//
+// prepared is P' for installing and ignored otherwise. A third
+// replacement is refused with ErrReplacementsExceeded; the worker then
+// marks the operation stuck.
+func ReplaceTarget(ctx context.Context, s release.Store, observed Observed, opID string, ref FenceRef, d *LocalDischarge, next Target, prepared string, authority FenceAuthority) (Observed, error) {
+ if observed.Absent || observed.Claim == nil {
+ return Observed{}, ErrWrongState
+ }
+ old := observed.Claim.Op.Intent["target_box"]
+ v, err := VerifyFence(ctx, s, authority, ref)
+ if err != nil {
+ return Observed{}, err
+ }
+ if old == "" || v.record.Box != old || v.record.Enrollment != observed.Claim.Op.Intent["target_enrollment"] || v.binding.Region != observed.Claim.Op.Intent["region"] {
+ return Observed{}, ErrFence
+ }
+ return Update(ctx, s, observed, func(c *Claim) error {
+ if err := requireRelocation(c, opID, PhaseStarting, PhaseReleasing, PhasePinning, PhaseInstalling, PhaseInstalled, PhaseProbing, PhaseProbeFailed); err != nil {
+ return err
+ }
+ if c.Op.Intent["target_box"] != old {
+ return ErrStaleOp
+ }
+ phase := c.Op.Phase
+ if err := fenceBox(c, old, ref, v.record.ObservedAt, d); err != nil {
+ return err
+ }
+ if err := replaceTargetSlot(c, next); err != nil {
+ return err
+ }
+ switch {
+ case phase == PhaseStarting || phase == PhaseReleasing || phase == PhasePinning:
+ return nil
+ case phase == PhaseInstalling:
+ if prepared == "" || prepared == c.Op.Journal["prepared"] || prepared == c.Holder.Generation {
+ return fmt.Errorf("%w: a replacement in installing builds a fresh P'", ErrRelocation)
+ }
+ return install(c, next.Box, prepared, PhaseInstalling)
+ case (phase == PhaseInstalled || phase == PhaseProbing) && !preparedOpened(c):
+ return install(c, next.Box, c.Op.Journal["prepared"], PhaseInstalled)
+ default:
+ // After P's first admission, or from probe-failed: nested
+ // failover from pinning, pinning from holder.generation as
+ // observed. A P no writer opened (a probe deadline before any
+ // admission) is recorded before it is dropped.
+ if err := abandonPrepared(c); err != nil {
+ return err
+ }
+ c.Op.Exec, c.Op.Phase = "none", PhasePinning
+ for _, k := range []string{"pin", "prepared", "prepared_authority", "local_install", "probe", "release", "release.phase"} {
+ delete(c.Op.Journal, k)
+ }
+ return nil
+ }
+ })
+}
+
+// FenceExhaustedTarget is the terminal transition for a target terminated
+// after the operation has used its replacements (§5.8 "The bound"; round 3
+// finding 5): it consumes the fence (the loss is retained, the box goes on
+// lost) and records stuck. While another box is op.exec (the live source
+// in starting or releasing; round 8 finding 1) nothing else changes: the
+// target's fence revokes nothing of the source's. Otherwise (op.exec is
+// the dead target, or none) it revokes the dead installation's authority (op.exec "none", the
+// target off the barrier, its receipt dropped), leaves the operation at
+// pinning with its pin and, when P was never opened, its remote build kept
+// for the operator's replacement, or a nested pinning when P was opened,
+// and records stuck in the same CAS. No third target is chosen. MarkStuck's
+// "never between installed and a probe" is for a live target; a dead one
+// has no probe to make.
+func FenceExhaustedTarget(ctx context.Context, s release.Store, observed Observed, opID string, ref FenceRef, d *LocalDischarge, authority FenceAuthority) (Observed, error) {
+ return fenceTargetStuck(ctx, s, observed, opID, ref, d, authority, "")
+}
+
+// FenceTargetWithoutSpare is FenceExhaustedTarget's transition for a
+// terminated target with replacements left but no eligible box to name
+// (rev 18, round 17 finding 2): the worker calls it once its bound has
+// passed. The fence is consumed in the same CAS (loss retained, the box
+// on lost, a dead executor's authority revoked), and the operation is
+// stuck for an operator with reason, never left waiting unbounded.
+func FenceTargetWithoutSpare(ctx context.Context, s release.Store, observed Observed, opID string, ref FenceRef, d *LocalDischarge, authority FenceAuthority, reason string) (Observed, error) {
+ if reason == "" {
+ return Observed{}, fmt.Errorf("%w: a stuck operation names its reason", ErrRelocation)
+ }
+ return fenceTargetStuck(ctx, s, observed, opID, ref, d, authority, reason)
+}
+
+// fenceTargetStuck is both: noSpare "" is the exhausted bound.
+func fenceTargetStuck(ctx context.Context, s release.Store, observed Observed, opID string, ref FenceRef, d *LocalDischarge, authority FenceAuthority, noSpare string) (Observed, error) {
+ if observed.Absent || observed.Claim == nil {
+ return Observed{}, ErrWrongState
+ }
+ old := observed.Claim.Op.Intent["target_box"]
+ v, err := VerifyFence(ctx, s, authority, ref)
+ if err != nil {
+ return Observed{}, err
+ }
+ if old == "" || v.record.Box != old || v.record.Enrollment != observed.Claim.Op.Intent["target_enrollment"] || v.binding.Region != observed.Claim.Op.Intent["region"] {
+ return Observed{}, ErrFence
+ }
+ return Update(ctx, s, observed, func(c *Claim) error {
+ if err := RequireOp(opID)(c); err != nil {
+ return err
+ }
+ if c.Op.Kind != KindRelocating || c.Op.Intent["target_box"] != old {
+ return ErrStaleOp
+ }
+ ensureJournal(c)
+ if c.Op.Journal["stuck"] != "" {
+ return ErrAlreadyRecorded
+ }
+ reason := noSpare
+ if reason == "" {
+ if journalUint(c.Op, "replacements") < MaxTargetReplacements {
+ return fmt.Errorf("%w: replacements remain; replace the target instead", ErrRelocation)
+ }
+ reason = fmt.Sprintf("%s terminated after %d target replacements", old, MaxTargetReplacements)
+ }
+ if err := fenceBox(c, old, ref, v.record.ObservedAt, d); err != nil {
+ return err
+ }
+ removeParticipant(c, old)
+ if c.Op.Exec != old && c.Op.Exec != "none" {
+ // The executor is another box (the live source in starting or
+ // releasing): the target's fence is no evidence against it.
+ // Its execution identity, phase and release records stay, so
+ // root still accepts its own stop (claimstop.go requires the
+ // requester to be op.exec); only stuck is new.
+ c.Op.Journal["stuck"] = reason
+ return nil
+ }
+ opened := preparedOpened(c)
+ c.Op.Exec, c.Op.Phase = "none", PhasePinning
+ keys := []string{"local_install", "probe", "release", "release.phase", "probing_since", "probe_failed_since"}
+ if opened {
+ // Nested: pinning again from holder.generation, as a replacement
+ // after P's first admission would.
+ keys = append(keys, "pin", "prepared", "prepared_authority")
+ }
+ // An unopened P is kept for the operator's replacement; it is on
+ // the abandoned list only if a later transition drops it.
+ for _, k := range keys {
+ delete(c.Op.Journal, k)
+ }
+ if c.Op.Journal["stuck"] == "" {
+ c.Op.Journal["stuck"] = reason
+ }
+ return nil
+ })
+}
+
+// MarkStuck stops the operation at its current phase for an operator
+// (`carlos ops host resolve`). It never resets the replacement count and
+// never invents evidence; every later relocation transition refuses.
+func MarkStuck(opID, reason string) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := RequireOp(opID)(c); err != nil {
+ return err
+ }
+ if c.Op.Kind != KindRelocating || reason == "" {
+ return ErrWrongState
+ }
+ ensureJournal(c)
+ if c.Op.Phase == PhaseInstalled && c.Op.Journal["local_install"] != "" && !preparedOpened(c) {
+ return fmt.Errorf("%w: never stop between installed and a probe", ErrRelocation)
+ }
+ if c.Op.Journal["stuck"] != "" {
+ return ErrAlreadyRecorded
+ }
+ c.Op.Journal["stuck"] = reason
+ return nil
+ }
+}
diff --git a/internal/hostclaim/relocation_test.go b/internal/hostclaim/relocation_test.go
new file mode 100644
index 000000000..46a043141
--- /dev/null
+++ b/internal/hostclaim/relocation_test.go
@@ -0,0 +1,1098 @@
+package hostclaim
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "reflect"
+ "slices"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/boxid"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/termfence"
+)
+
+const (
+ relSource = "adi-knitting"
+ relTarget = "bri-knitting"
+ relSpare = "cai-knitting"
+ relSpare2 = "dov-knitting"
+)
+
+var relObserved = time.Date(2026, 10, 7, 9, 0, 0, 0, time.UTC)
+
+func relEnrollment(box string) string { return strings.Repeat(string(box[0]), 32) }
+
+// relAuthority accepts any fence record for a box in this region.
+type relAuthority struct{}
+
+func (relAuthority) ValidateFence(_ context.Context, f termfence.Record) (FenceBinding, error) {
+ if f.Enrollment != relEnrollment(f.Box) {
+ return FenceBinding{}, ErrFence
+ }
+ return FenceBinding{Region: "ie", RosterRevision: 9, RosterETag: "roster"}, nil
+}
+
+type relReleases struct{ err error }
+
+func (r relReleases) VerifyRelease(context.Context, Claim, Release) error { return r.err }
+
+func putFence(t *testing.T, s release.Store, box string) FenceRef {
+ t.Helper()
+ f := termfence.Record{V: 1, Box: box, Enrollment: relEnrollment(box), State: "terminated",
+ Cloud: boxid.Cloud{Provider: "aws", Account: "123", Region: "eu-west-1", InstanceID: "i-" + box},
+ ObservedAt: relObserved.Format(time.RFC3339Nano), Via: "describe", RequestID: "req",
+ WrittenBy: termfence.WrittenBy{Box: "worker", Enrollment: strings.Repeat("e", 32)}}
+ raw, err := json.Marshal(f)
+ if err != nil {
+ t.Fatal(err)
+ }
+ sum := sha256.Sum256(raw)
+ ref := FenceRef{Enrollment: f.Enrollment, Key: "control/box-fences/" + f.Enrollment + ".json", SHA256: hex.EncodeToString(sum[:])}
+ if _, err := s.Put(context.Background(), ref.Key, raw, ""); err != nil && !errors.Is(err, release.ErrCASConflict) {
+ t.Fatal(err)
+ }
+ return ref
+}
+
+// relServing is a serving claim on relSource whose source has admitted one
+// cold writer session (generation g1).
+func relServing(t *testing.T) (release.Store, Observed) {
+ t.Helper()
+ ctx := context.Background()
+ s := release.NewDirStore(t.TempDir())
+ o := servingClaim(t, ctx, s)
+ o, err := Update(ctx, s, o, AdmitWriter(Admission{Box: relSource, Mode: "cold", NewGeneration: "g1", Region: "ie", KeepStamp: true}))
+ if err != nil {
+ t.Fatal(err)
+ }
+ return s, o
+}
+
+func relStart(reason string, fence *FenceRef) RelocationStart {
+ return RelocationStart{OpID: "rel-1", By: "worker", Now: relObserved, Region: "ie", Reason: reason,
+ SourceBox: relSource, SourceEnrollment: relEnrollment(relSource),
+ TargetBox: relTarget, TargetEnrollment: relEnrollment(relTarget),
+ DirectoryRevision: 9, AdmissionID: "adm-1", Fence: fence}
+}
+
+func step(t *testing.T, s release.Store, o Observed, tr func(*Claim) error) Observed {
+ t.Helper()
+ next, err := Update(context.Background(), s, o, tr)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return next
+}
+
+func ack(box string) func(*Claim) error {
+ return AcknowledgeExecution(box, ExecutionAcknowledgement{Op: "rel-1", Enrollment: relEnrollment(box), BootID: "boot", ControllerID: "ctl", LeaseETag: "lease", Expires: relObserved.Add(time.Hour)})
+}
+
+// relPin is a pin selected against o: its key names o's claim generation.
+func relPin(o Observed, maxTXID uint64, seq int64, at time.Time) RecoveryPin {
+ return RecoveryPin{Key: RecoveryPinKey(o.Claim.Host, "rel-1", o.Claim.Gen), SHA256: strings.Repeat("a", 64), Generation: "g1", MaxTXID: maxTXID, FilesManifest: "instances/x/files/g1/manifest.json", FilesSeq: seq, PinnedAt: at, Empty: maxTXID == 0}
+}
+
+func relPrepared(gen string, maxTXID uint64, filesSeq int64) PreparedAuthority {
+ a := PreparedAuthority{Generation: gen, SnapshotTXID: maxTXID, FilesSeq: 0, SourceFilesSeq: filesSeq, FilesManifestSHA256: strings.Repeat("f", 64)}
+ if maxTXID > 0 {
+ a.InstallManifest, a.SHA256 = "m", strings.Repeat("c", 64)
+ }
+ return a
+}
+
+func localInstall(c *Claim) LocalInstall {
+ return LocalInstall{Box: c.Op.Exec, Enrollment: relEnrollment(c.Op.Exec), InstallStamp: journalUint(c.Op, "install_stamp"), FilesManifestVersion: 3, JournalSHA256: strings.Repeat("b", 64)}
+}
+
+// toProbing drives an operation at pinning (with its pin) through
+// installing, installed and the local install receipt into probing.
+func toProbing(t *testing.T, s release.Store, o Observed, prepared string, maxTXID uint64) Observed {
+ t.Helper()
+ target := Target{Box: o.Claim.Op.Intent["target_box"], Enrollment: o.Claim.Op.Intent["target_enrollment"]}
+ o = step(t, s, o, ToInstalling("rel-1", target, prepared))
+ var pin RecoveryPin
+ readJournal(o.Claim.Op, "pin", &pin)
+ o = step(t, s, o, ToInstalled("rel-1", relPrepared(prepared, maxTXID, pin.FilesSeq)))
+ o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim)))
+ return step(t, s, o, ToProbing("rel-1", relObserved))
+}
+
+func TestRelocationDrainEndToEnd(t *testing.T) {
+ ctx := context.Background()
+ s, o := relServing(t)
+ o, err := StartRelocation(ctx, s, o, relStart(ReasonDrain, nil), relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if c := o.Claim; c.Op.Exec != relSource || c.Op.Phase != PhaseStarting || !slices.Equal(c.Op.Participants, []string{relSource, relTarget}) || c.Op.Intent["stamp_from"] == "" {
+ t.Fatalf("start: %+v", c.Op)
+ }
+ if _, err := Update(ctx, s, o, ToReleasing("rel-1", relObserved)); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("barrier without acknowledgements: %v", err)
+ }
+ o = step(t, s, o, ack(relSource))
+ o = step(t, s, o, ack(relTarget))
+ o = step(t, s, o, ToReleasing("rel-1", relObserved))
+ if o.Claim.Op.Journal["releasing_since"] == "" {
+ t.Fatal("releasing did not start its deadline")
+ }
+ h := o.Claim.Holder
+ rel := Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "control/stop/r.json", ReceiptSHA256: strings.Repeat("d", 64)}
+ o = step(t, s, o, JournalRelease("rel-1", rel))
+ if _, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{err: errors.New("bad receipt")}, nil); err == nil {
+ t.Fatal("pinned on an unverified receipt")
+ }
+ if o, err = PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil); err != nil {
+ t.Fatal(err)
+ }
+ stamp := o.Claim.Holder.Stamp
+ if o.Claim.Op.Exec != "none" || o.Claim.Op.Phase != PhasePinning {
+ t.Fatalf("pinning: %+v", o.Claim.Op)
+ }
+ if _, err := Update(ctx, s, o, JournalPin("rel-1", relPin(o, 39, 7, relObserved))); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("pin short of the release: %v", err)
+ }
+ stale := relPin(o, 40, 7, relObserved)
+ stale.Key = RecoveryPinKey(o.Claim.Host, "rel-1", o.Claim.Gen-1)
+ if _, err := Update(ctx, s, o, JournalPin("rel-1", stale)); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("a pin selected against an earlier observation was journaled: %v", err)
+ }
+ o = step(t, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved)))
+ if o.Claim.Holder.Stamp != stamp {
+ t.Fatal("pinning changed the stamp")
+ }
+ o = step(t, s, o, ToInstalling("rel-1", Target{relTarget, relEnrollment(relTarget)}, "P1"))
+ for name, a := range map[string]PreparedAuthority{
+ "no files manifest": {Generation: "P1", InstallManifest: "m", SHA256: strings.Repeat("c", 64), SnapshotTXID: 40},
+ "another files seq": {Generation: "P1", InstallManifest: "m", SHA256: strings.Repeat("c", 64), SnapshotTXID: 40, FilesSeq: 6, FilesManifestSHA256: strings.Repeat("f", 64)},
+ "no install manifest": {Generation: "P1", SnapshotTXID: 40, FilesSeq: 7, FilesManifestSHA256: strings.Repeat("f", 64)},
+ "another snapshot txid": {Generation: "P1", InstallManifest: "m", SHA256: strings.Repeat("c", 64), SnapshotTXID: 41, FilesSeq: 7, FilesManifestSHA256: strings.Repeat("f", 64)},
+ } {
+ if _, err := Update(ctx, s, o, ToInstalled("rel-1", a)); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("%s: installed on an incomplete build: %v", name, err)
+ }
+ }
+ o = step(t, s, o, ToInstalled("rel-1", relPrepared("P1", 40, 7)))
+ o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim)))
+ o = step(t, s, o, ToProbing("rel-1", relObserved))
+ install := journalUint(o.Claim.Op, "install_stamp")
+ if o.Claim.Op.Exec != relTarget || o.Claim.Holder.Stamp != install || !slices.Contains(o.Claim.Touched.Boxes, relTarget) {
+ t.Fatalf("installation: exec %s stamp %d install %d", o.Claim.Op.Exec, o.Claim.Holder.Stamp, install)
+ }
+ o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
+ Box: relTarget, Mode: "cold", NewGeneration: "P1", KeepStamp: true, Coverage: "released", Region: "ie", PredecessorRegion: "ie"}))
+ last := o.Claim.Holder.Admissions[len(o.Claim.Holder.Admissions)-1]
+ if last.Coverage != "released" || last.WritesLost != nil || o.Claim.Holder.Stamp != install {
+ t.Fatalf("released admission: %+v stamp %d", last, o.Claim.Holder.Stamp)
+ }
+ o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relTarget, Session: o.Claim.Holder.Session}))
+ o = step(t, s, o, CompleteRelocation("rel-1", counter()))
+ c := o.Claim
+ if c.Op.Kind != KindServing || c.Op.ID != "" || c.Op.Exec != relTarget || !slices.Equal(c.Op.Participants, []string{relTarget}) {
+ t.Fatalf("serving: %+v", c.Op)
+ }
+ if len(c.Obligations) != 1 || c.Obligations[0].Kind != "park-local" || c.Obligations[0].Box != relSource || c.Obligations[0].Op != "rel-1" {
+ t.Fatalf("the source's park-local: %+v", c.Obligations)
+ }
+}
+
+func TestRelocationFailoverCitesTheFenceAndDeclaresTheLoss(t *testing.T) {
+ ctx := context.Background()
+ for _, qualified := range []bool{false, true} {
+ s, o := relServing(t)
+ o = step(t, s, o, func(c *Claim) error { // a chore the dead source already owed
+ c.Obligations = append(c.Obligations, Obligation{ID: "old", Op: "old-op", Kind: "park-local", Box: relSource, Retirement: "r", Owner: "box:" + relSource})
+ return nil
+ })
+ ref := putFence(t, s, relSource)
+ sourceSession := o.Claim.Holder.Session
+ in := relStart(ReasonFailover, &ref)
+ in.Discharge = &LocalDischarge{Qualified: qualified, Records: []string{"control/box-volumes/x/1.json"}}
+ o, err := StartRelocation(ctx, s, o, in, relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ c := o.Claim
+ if c.Op.Exec != "none" || !slices.Equal(c.Op.Participants, []string{relTarget}) || !slices.Contains(c.Lost, relSource) || c.Op.Journal["fence."+relSource] == "" {
+ t.Fatalf("failover start: %+v lost %v", c.Op, c.Lost)
+ }
+ if owes := slices.ContainsFunc(c.Obligations, func(o Obligation) bool { return o.Box == relSource }); owes == qualified {
+ t.Fatalf("discharge qualified=%v left chores %+v", qualified, c.Obligations)
+ }
+ if _, err := Update(ctx, s, o, ToReleasing("rel-1", relObserved)); err == nil {
+ t.Fatal("a failover entered releasing")
+ }
+ o = step(t, s, o, ack(relTarget))
+ o = step(t, s, o, PinFailover("rel-1"))
+ if _, err := Update(ctx, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved.Add(29*time.Second)))); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("pinned inside the 30-second window: %v", err)
+ }
+ o = step(t, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved.Add(FailoverPinDelay))))
+ o = toProbing(t, s, o, "P1", 40)
+ probe := Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
+ Box: relTarget, Mode: "cold", NewGeneration: "P1", KeepStamp: true, Coverage: "released", Region: "ie", PredecessorRegion: "ie"}
+ if _, err := Update(ctx, s, o, AdmitWriter(probe)); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("a failover admission claimed released coverage: %v", err)
+ }
+ probe.Coverage = ""
+ o = step(t, s, o, AdmitWriter(probe))
+ last := o.Claim.Holder.Admissions[len(o.Claim.Holder.Admissions)-1]
+ if last.WritesLost == nil || last.WritesLost.Box != relSource || last.WritesLost.Generation != "g1" {
+ t.Fatalf("failover must declare the source's session: %+v", last)
+ }
+ o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relTarget, Session: o.Claim.Holder.Session}))
+ o = step(t, s, o, CompleteRelocation("rel-1", counter()))
+ owes := slices.ContainsFunc(o.Claim.Obligations, func(ob Obligation) bool { return ob.Kind == "park-local" && ob.Box == relSource })
+ if owes == qualified {
+ t.Fatalf("qualified=%v: source chores at serving %+v", qualified, o.Claim.Obligations)
+ }
+ // D4: the loss outlives the operation, with the console still to
+ // materialize it (nothing here stands in for the console).
+ c = o.Claim
+ if len(c.EvidenceRefs) != 1 || len(c.LossRefs) != 1 || c.EvidenceRefs[0].ID != c.LossRefs[0] {
+ t.Fatalf("retained losses after serving: %+v refs %v", c.EvidenceRefs, c.LossRefs)
+ }
+ l := c.EvidenceRefs[0]
+ if l.Box != relSource || l.Fence != ref || l.ObservedAt != relObserved.Format(time.RFC3339Nano) || l.Op != "rel-1" || l.Role != "source" ||
+ l.Session != sourceSession || l.Generation != "g1" || l.Discharge.Qualified != qualified || l.Timers != TimersUnknown || l.Reported {
+ t.Fatalf("retained loss: %+v", l)
+ }
+ if c.Op.Journal != nil || !slices.Contains(c.Lost, relSource) {
+ t.Fatalf("serving kept the journal, or lost the box: %+v %v", c.Op, c.Lost)
+ }
+ }
+}
+
+func TestRelocationSourceLossBecomesFailover(t *testing.T) {
+ ctx := context.Background()
+ for _, tc := range []struct {
+ name string
+ prepare func(*testing.T, release.Store, Observed) Observed
+ wantPhase string
+ wantExec string
+ wantParts []string
+ }{
+ {"starting before acknowledging", func(t *testing.T, s release.Store, o Observed) Observed { return o }, PhaseStarting, "none", []string{relTarget}},
+ {"starting after acknowledging", func(t *testing.T, s release.Store, o Observed) Observed { return step(t, s, o, ack(relSource)) }, PhaseStarting, "none", []string{relTarget}},
+ {"releasing before its receipt", func(t *testing.T, s release.Store, o Observed) Observed {
+ o = step(t, s, step(t, s, o, ack(relSource)), ack(relTarget))
+ return step(t, s, o, ToReleasing("rel-1", relObserved))
+ }, PhasePinning, "none", []string{relTarget}},
+ {"releasing after its receipt", func(t *testing.T, s release.Store, o Observed) Observed {
+ o = step(t, s, step(t, s, o, ack(relSource)), ack(relTarget))
+ o = step(t, s, o, ToReleasing("rel-1", relObserved))
+ h := o.Claim.Holder
+ return step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 1, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)}))
+ }, PhaseReleasing, relSource, []string{relSource, relTarget}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ s, o := relServing(t)
+ o, err := StartRelocation(ctx, s, o, relStart(ReasonDrain, nil), relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ o = tc.prepare(t, s, o)
+ stamp := o.Claim.Holder.Stamp
+ o, err = FenceSource(ctx, s, o, "rel-1", putFence(t, s, relSource), nil, relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ c := o.Claim
+ if c.Op.Phase != tc.wantPhase || c.Op.Exec != tc.wantExec || !slices.Equal(c.Op.Participants, tc.wantParts) || c.Holder.Stamp != stamp || !slices.Contains(c.Lost, relSource) {
+ t.Fatalf("phase %s exec %s parts %v stamp %d lost %v", c.Op.Phase, c.Op.Exec, c.Op.Participants, c.Holder.Stamp, c.Lost)
+ }
+ if c.Op.Intent["reason"] != ReasonDrain {
+ t.Fatal("the intent was rewritten")
+ }
+ })
+ }
+}
+
+// relAtPhase starts a drain and drives it to phase. "probing+P" is probing
+// after the first admission opened P.
+func relAtPhase(t *testing.T, phase string) (release.Store, Observed) {
+ t.Helper()
+ ctx := context.Background()
+ s, o := relServing(t)
+ o, err := StartRelocation(ctx, s, o, relStart(ReasonDrain, nil), relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ o = step(t, s, step(t, s, o, ack(relSource)), ack(relTarget))
+ if phase == PhaseStarting {
+ return s, o
+ }
+ o = step(t, s, o, ToReleasing("rel-1", relObserved))
+ if phase == PhaseReleasing {
+ return s, o
+ }
+ h := o.Claim.Holder
+ o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)}))
+ if o, err = PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil); err != nil {
+ t.Fatal(err)
+ }
+ o = step(t, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved)))
+ if phase == PhasePinning {
+ return s, o
+ }
+ o = step(t, s, o, ToInstalling("rel-1", Target{relTarget, relEnrollment(relTarget)}, "P1"))
+ if phase == PhaseInstalling {
+ return s, o
+ }
+ o = step(t, s, o, ToInstalled("rel-1", relPrepared("P1", 40, 7)))
+ if phase == PhaseInstalled {
+ return s, o
+ }
+ o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim)))
+ o = step(t, s, o, ToProbing("rel-1", relObserved))
+ if phase == PhaseProbing {
+ return s, o
+ }
+ o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
+ Box: relTarget, Mode: "cold", NewGeneration: "P1", KeepStamp: true, Coverage: "released", Region: "ie", PredecessorRegion: "ie"}))
+ return s, o
+}
+
+func TestRelocationReplacementTable(t *testing.T) {
+ ctx := context.Background()
+ for _, tc := range []struct {
+ at, wantPhase, wantExec string
+ newStamp, keepP bool
+ }{
+ {PhaseStarting, PhaseStarting, relSource, false, false},
+ {PhaseReleasing, PhaseReleasing, relSource, false, false},
+ {PhasePinning, PhasePinning, "none", false, false},
+ {PhaseInstalling, PhaseInstalling, relSpare, true, false},
+ {PhaseInstalled, PhaseInstalled, relSpare, true, true},
+ {PhaseProbing, PhaseInstalled, relSpare, true, true},
+ {"probing+P", PhasePinning, "none", false, false},
+ } {
+ t.Run(tc.at, func(t *testing.T) {
+ s, o := relAtPhase(t, tc.at)
+ before := o.Claim.Holder.Stamp
+ o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "P2", relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ c := o.Claim
+ if c.Op.Phase != tc.wantPhase || c.Op.Exec != tc.wantExec || c.Op.Intent["target_box"] != relSpare || !slices.Contains(c.Lost, relTarget) || slices.Contains(c.Op.Participants, relTarget) {
+ t.Fatalf("phase %s exec %s intent %v lost %v parts %v", c.Op.Phase, c.Op.Exec, c.Op.Intent, c.Lost, c.Op.Participants)
+ }
+ if (c.Holder.Stamp != before) != tc.newStamp {
+ t.Fatalf("stamp %d -> %d", before, c.Holder.Stamp)
+ }
+ if tc.keepP && c.Op.Journal["prepared"] != "P1" {
+ t.Fatalf("the unopened P must be reused: %q", c.Op.Journal["prepared"])
+ }
+ if c.Op.Journal["local_install"] != "" {
+ t.Fatal("a replacement inherited its predecessor's receipt")
+ }
+ if tc.wantPhase == PhaseInstalled {
+ if _, err := Update(ctx, s, o, ToProbing("rel-1", relObserved)); err == nil {
+ t.Fatal("probing without the replacement's own local install")
+ }
+ }
+ })
+ }
+}
+
+func TestRelocationTwoReplacementsThenStuck(t *testing.T) {
+ ctx := context.Background()
+ s, o := relAtPhase(t, PhasePinning)
+ var err error
+ for _, next := range []string{relSpare, relSpare2} {
+ old := o.Claim.Op.Intent["target_box"]
+ if o, err = ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, old), nil, Target{next, relEnrollment(next)}, "", relAuthority{}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ if _, err = ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relSpare2), nil, Target{"eve-knitting", relEnrollment("eve-knitting")}, "", relAuthority{}); !errors.Is(err, ErrReplacementsExceeded) {
+ t.Fatalf("third replacement: %v", err)
+ }
+ o = step(t, s, o, MarkStuck("rel-1", "two target replacements used"))
+ if journalUint(o.Claim.Op, "replacements") != 2 {
+ t.Fatal("stuck reset the count")
+ }
+ if _, err := Update(ctx, s, o, ToInstalling("rel-1", Target{relSpare2, relEnrollment(relSpare2)}, "P9")); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("a stuck operation moved: %v", err)
+ }
+}
+
+func TestRelocationReplacementInReleasingAcknowledgesBeforePinning(t *testing.T) {
+ ctx := context.Background()
+ s, o := relAtPhase(t, PhaseReleasing)
+ h := o.Claim.Holder
+ o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)}))
+ o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "", relAuthority{})
+ if err != nil || o.Claim.Op.Phase != PhaseReleasing {
+ t.Fatalf("replacement in releasing: %v %+v", err, o.Claim.Op)
+ }
+ if _, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("left releasing before the replacement acknowledged: %v", err)
+ }
+ o = step(t, s, o, ack(relSpare))
+ if o, err = PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil); err != nil || o.Claim.Op.Phase != PhasePinning {
+ t.Fatalf("after the acknowledgement: %v %+v", err, o.Claim.Op)
+ }
+}
+
+func TestRelocationExhaustedTargetIsFencedIntoAStuckPinning(t *testing.T) {
+ ctx := context.Background()
+ exhaust := func(t *testing.T, s release.Store, o Observed) Observed {
+ var err error
+ for _, next := range []string{relSpare, relSpare2} {
+ old := o.Claim.Op.Intent["target_box"]
+ if o, err = ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, old), nil, Target{next, relEnrollment(next)}, "", relAuthority{}); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return o
+ }
+ t.Run("before exhaustion it is a replacement", func(t *testing.T) {
+ s, o := relAtPhase(t, PhasePinning)
+ if _, err := FenceExhaustedTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, relAuthority{}); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("fenced as exhausted with replacements left: %v", err)
+ }
+ })
+ // Round 8 finding 1: while the live source is still op.exec, the
+ // target's fence consumes the target and records stuck; the source's
+ // execution identity, phase and release evidence stay.
+ for _, phase := range []string{PhaseStarting, PhaseReleasing} {
+ t.Run("dead while the source executes in "+phase, func(t *testing.T) {
+ s, o := relAtPhase(t, phase)
+ if phase == PhaseReleasing {
+ h := o.Claim.Holder
+ o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, ManifestSeq: 7, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)}))
+ }
+ o = exhaust(t, s, o)
+ before := o.Claim
+ ref := putFence(t, s, relSpare2)
+ o, err := FenceExhaustedTarget(ctx, s, o, "rel-1", ref, nil, relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ c := o.Claim
+ if c.Op.Phase != phase || c.Op.Exec != relSource || c.Holder.Stamp != before.Holder.Stamp || c.Op.Journal["stuck"] == "" ||
+ !slices.Contains(c.Op.Participants, relSource) || slices.Contains(c.Op.Participants, relSpare2) || !slices.Contains(c.Lost, relSpare2) {
+ t.Fatalf("the live source was revoked: %+v lost %v", c.Op, c.Lost)
+ }
+ for _, k := range []string{"release", "release.phase", "releasing_since", "exec_ack." + relSource} {
+ if c.Op.Journal[k] != before.Op.Journal[k] {
+ t.Fatalf("%s changed: %q -> %q", k, before.Op.Journal[k], c.Op.Journal[k])
+ }
+ }
+ if phase == PhaseReleasing && c.Op.Journal["release"] == "" {
+ t.Fatal("the source's release evidence was dropped")
+ }
+ if l := c.EvidenceRefs[len(c.EvidenceRefs)-1]; l.Box != relSpare2 || l.Fence != ref {
+ t.Fatalf("the dead target's loss: %+v", c.EvidenceRefs)
+ }
+ if err := c.Validate(); err != nil {
+ t.Fatal(err)
+ }
+ })
+ }
+ t.Run("dead after its local install, P unopened", func(t *testing.T) {
+ s, o := relAtPhase(t, PhasePinning)
+ o = exhaust(t, s, o)
+ o = step(t, s, o, ack(relSpare2))
+ o = step(t, s, o, ToInstalling("rel-1", Target{relSpare2, relEnrollment(relSpare2)}, "P1"))
+ o = step(t, s, o, ToInstalled("rel-1", relPrepared("P1", 40, 7)))
+ o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim)))
+ if _, err := Update(ctx, s, o, MarkStuck("rel-1", "x")); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("MarkStuck between installed and a probe: %v", err)
+ }
+ ref := putFence(t, s, relSpare2)
+ o, err := FenceExhaustedTarget(ctx, s, o, "rel-1", ref, nil, relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ c := o.Claim
+ if c.Op.Phase != PhasePinning || c.Op.Exec != "none" || c.Op.Journal["stuck"] == "" || slices.Contains(c.Op.Participants, relSpare2) || !slices.Contains(c.Lost, relSpare2) ||
+ c.Op.Journal["local_install"] != "" || c.Op.Journal["pin"] == "" || c.Op.Journal["prepared"] != "P1" || c.Op.Journal["prepared_authority"] == "" {
+ t.Fatalf("exhausted and dead: %+v lost %v", c.Op, c.Lost)
+ }
+ if len(c.EvidenceRefs) != 3 || c.EvidenceRefs[2].Box != relSpare2 || c.EvidenceRefs[2].Fence != ref {
+ t.Fatalf("the dead target's loss: %+v", c.EvidenceRefs)
+ }
+ if _, err := Update(ctx, s, o, ToInstalling("rel-1", Target{relSpare2, relEnrollment(relSpare2)}, "P2")); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("a stuck operation moved: %v", err)
+ }
+ if _, err := FenceExhaustedTarget(ctx, s, o, "rel-1", ref, nil, relAuthority{}); err == nil {
+ t.Fatal("the fence was consumed twice")
+ }
+ })
+ t.Run("dead after P was opened", func(t *testing.T) {
+ s, o := relAtPhase(t, "probing+P")
+ o = exhaust(t, s, o) // after P's admission each replacement is a nested pinning
+ o = step(t, s, o, ack(relSpare2))
+ o = step(t, s, o, JournalPin("rel-1", relPin(o, 60, 9, relObserved.Add(FailoverPinDelay))))
+ o = step(t, s, o, ToInstalling("rel-1", Target{relSpare2, relEnrollment(relSpare2)}, "P3"))
+ o = step(t, s, o, ToInstalled("rel-1", relPrepared("P3", 60, 9)))
+ o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim)))
+ o = step(t, s, o, ToProbing("rel-1", relObserved))
+ o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
+ Box: relSpare2, Mode: "cold", NewGeneration: "P3", KeepStamp: true, Region: "ie", PredecessorRegion: "ie"}))
+ o, err := FenceExhaustedTarget(ctx, s, o, "rel-1", putFence(t, s, relSpare2), nil, relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ c := o.Claim
+ if c.Op.Phase != PhasePinning || c.Op.Exec != "none" || c.Op.Journal["stuck"] == "" || c.Op.Journal["pin"] != "" || c.Op.Journal["prepared"] != "" {
+ t.Fatalf("nested stuck pinning: %+v", c.Op)
+ }
+ })
+}
+
+func TestRelocationProbeFailureIsANestedRelocation(t *testing.T) {
+ ctx := context.Background()
+ s, o := relAtPhase(t, "probing+P")
+ o = step(t, s, o, ToProbeFailed("rel-1", relObserved))
+ if o.Claim.Op.Journal["probe_failed_since"] == "" {
+ t.Fatal("probe-failed did not start its release deadline")
+ }
+ h := o.Claim.Holder
+ rel := Release{ReleasedBy: relTarget, Enrollment: relEnrollment(relTarget), Generation: h.Generation, Session: h.Session, FinalTXID: 55, ManifestSeq: 9, Receipt: "r2", ReceiptSHA256: strings.Repeat("d", 64)}
+ o = step(t, s, o, JournalRelease("rel-1", rel))
+ if _, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, &Target{relTarget, relEnrollment(relTarget)}); err == nil {
+ t.Fatal("the failed target was named again")
+ }
+ o, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, &Target{relSpare, relEnrollment(relSpare)})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if o.Claim.Op.Phase != PhasePinning || o.Claim.Op.Exec != "none" {
+ t.Fatalf("nested relocation: %+v", o.Claim.Op)
+ }
+ if _, err := Update(ctx, s, o, JournalPin("rel-1", relPin(o, 54, 9, relObserved))); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("pin lost the probe's writes: %v", err)
+ }
+ o = step(t, s, o, JournalPin("rel-1", relPin(o, 55, 9, relObserved)))
+ o = step(t, s, o, ack(relSpare))
+ o = toProbing(t, s, o, "P2", 55)
+ // released coverage on P2's opening: the predecessor is the failed
+ // target's probe session, whose release this operation journaled.
+ o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
+ Box: relSpare, Mode: "cold", NewGeneration: "P2", KeepStamp: true, Coverage: "released", Region: "ie", PredecessorRegion: "ie"}))
+ o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relSpare, Session: o.Claim.Holder.Session}))
+ o = step(t, s, o, CompleteRelocation("rel-1", counter()))
+ var owed []string
+ for _, ob := range o.Claim.Obligations {
+ owed = append(owed, ob.Box)
+ }
+ slices.Sort(owed)
+ if !slices.Equal(owed, []string{relSource, relTarget}) {
+ t.Fatalf("park-local owed by the source and the abandoned target: %v", owed)
+ }
+}
+
+func TestRelocationAdmissionRules(t *testing.T) {
+ ctx := context.Background()
+ probe := func(o Observed, gen, coverage string) Admission {
+ return Admission{ObservedOpID: "rel-1", ObservedPhase: o.Claim.Op.Phase, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
+ Box: o.Claim.Op.Exec, Mode: "cold", NewGeneration: gen, Coverage: coverage, Region: "ie", PredecessorRegion: "ie"}
+ }
+ for _, tc := range []struct {
+ name, at string
+ admit func(Observed) Admission
+ }{
+ {"the target admits in installing", PhaseInstalling, func(o Observed) Admission { return probe(o, "P1", "") }},
+ {"the target admits in installed", PhaseInstalled, func(o Observed) Admission { return probe(o, "P1", "") }},
+ {"the first probe skips P", PhaseProbing, func(o Observed) Admission { return probe(o, "other", "") }},
+ {"P is opened twice", "probing+P", func(o Observed) Admission { return probe(o, "P1", "") }},
+ {"released after P was opened", "probing+P", func(o Observed) Admission { return probe(o, "P3", "released") }},
+ {"warm probe", PhaseProbing, func(o Observed) Admission {
+ a := probe(o, "", "")
+ a.Mode, a.PredecessorCertified = "warm", true
+ return a
+ }},
+ {"the source claims released", PhaseStarting, func(o Observed) Admission { return probe(o, "g2", "released") }},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ s, o := relAtPhase(t, tc.at)
+ if _, err := Update(ctx, s, o, AdmitWriter(tc.admit(o))); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("admitted: %v", err)
+ }
+ })
+ }
+ t.Run("released names another predecessor", func(t *testing.T) {
+ s, o := relAtPhase(t, PhaseProbing)
+ o.Claim.Holder.Admissions[len(o.Claim.Holder.Admissions)-1].Session++
+ if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "P1", "released"))); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("admitted: %v", err)
+ }
+ })
+ t.Run("released names another predecessor box", func(t *testing.T) {
+ s, o := relAtPhase(t, PhaseProbing)
+ o.Claim.Holder.Admissions[len(o.Claim.Holder.Admissions)-1].Box = relSpare
+ if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "P1", "released"))); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("admitted: %v", err)
+ }
+ })
+ t.Run("a third attempt reopens P", func(t *testing.T) {
+ s, o := relAtPhase(t, "probing+P")
+ o = step(t, s, o, AdmitWriter(probe(o, "P3", ""))) // attempt two: a fresh generation
+ if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "P1", ""))); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("P was opened again: %v", err)
+ }
+ })
+ t.Run("released outside relocating", func(t *testing.T) {
+ s, o := relServing(t)
+ if _, err := Update(ctx, s, o, AdmitWriter(Admission{ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation, Box: relSource, Mode: "cold", NewGeneration: "g2", Coverage: "released", Region: "ie"})); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("admitted: %v", err)
+ }
+ })
+ t.Run("the source wakes in starting", func(t *testing.T) {
+ s, o := relAtPhase(t, PhaseStarting)
+ if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "g2", ""))); err != nil {
+ t.Fatal(err)
+ }
+ })
+ t.Run("a stuck source does not wake", func(t *testing.T) {
+ s, o := relAtPhase(t, PhaseStarting)
+ o = step(t, s, o, MarkStuck("rel-1", "operator"))
+ if _, err := Update(ctx, s, o, AdmitWriter(probe(o, "g2", ""))); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("admitted: %v", err)
+ }
+ })
+}
+
+func TestRelocationEmptyPinNeedsItsEvidence(t *testing.T) {
+ ctx := context.Background()
+ t.Run("a release at txid 0 vouches", func(t *testing.T) {
+ s, o := relAtPhase(t, PhaseReleasing)
+ h := o.Claim.Holder
+ o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 0, ManifestSeq: 0, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)}))
+ o, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ o = step(t, s, o, JournalPin("rel-1", relPin(o, 0, 0, relObserved)))
+ o = step(t, s, o, ToInstalling("rel-1", Target{relTarget, relEnrollment(relTarget)}, "P1"))
+ if _, err := Update(ctx, s, o, ToInstalled("rel-1", relPrepared("P1", 40, 0))); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("an empty pin accepted a database build: %v", err)
+ }
+ if _, err := Update(ctx, s, o, ToInstalled("rel-1", PreparedAuthority{Generation: "P1", FilesSeq: 0})); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("an empty P without its files manifest: %v", err)
+ }
+ step(t, s, o, ToInstalled("rel-1", relPrepared("P1", 0, 0)))
+ })
+ t.Run("a release at txid 40 does not", func(t *testing.T) {
+ s, o := relAtPhase(t, PhaseReleasing)
+ h := o.Claim.Holder
+ o = step(t, s, o, JournalRelease("rel-1", Release{ReleasedBy: relSource, Enrollment: relEnrollment(relSource), Generation: h.Generation, Session: h.Session, FinalTXID: 40, Receipt: "r", ReceiptSHA256: strings.Repeat("d", 64)}))
+ o, err := PinAfterRelease(ctx, s, o, "rel-1", relReleases{}, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := Update(ctx, s, o, JournalPin("rel-1", relPin(o, 0, 0, relObserved))); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("an empty pin against a release at 40: %v", err)
+ }
+ })
+ t.Run("a failover of a host that had a writer does not", func(t *testing.T) {
+ s, o := relServing(t)
+ ref := putFence(t, s, relSource)
+ o, err := StartRelocation(ctx, s, o, relStart(ReasonFailover, &ref), relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ o = step(t, s, step(t, s, o, ack(relTarget)), PinFailover("rel-1"))
+ if _, err := Update(ctx, s, o, JournalPin("rel-1", relPin(o, 0, 0, relObserved.Add(FailoverPinDelay)))); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("a listing that found nothing was taken as empty: %v", err)
+ }
+ })
+ t.Run("a failover of a never-woken host does", func(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ o := servingClaim(t, ctx, s) // session 0: no writer was ever admitted
+ ref := putFence(t, s, relSource)
+ o, err := StartRelocation(ctx, s, o, relStart(ReasonFailover, &ref), relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ o = step(t, s, step(t, s, o, ack(relTarget)), PinFailover("rel-1"))
+ step(t, s, o, JournalPin("rel-1", relPin(o, 0, 0, relObserved.Add(FailoverPinDelay))))
+ })
+}
+
+func TestRelocationProbeAttemptsAreBounded(t *testing.T) {
+ ctx := context.Background()
+ s, o := relAtPhase(t, PhaseProbing)
+ for i := 0; i < MaxProbeAttempts; i++ {
+ o = step(t, s, o, BeginProbeAttempt("rel-1", relTarget))
+ }
+ if journalUint(o.Claim.Op, "probe_attempts."+relTarget) != MaxProbeAttempts {
+ t.Fatalf("count: %v", o.Claim.Op.Journal)
+ }
+ if _, err := Update(ctx, s, o, BeginProbeAttempt("rel-1", relTarget)); !errors.Is(err, ErrProbeAttemptsExhausted) {
+ t.Fatalf("a fourth attempt: %v", err)
+ }
+ if _, err := Update(ctx, s, o, BeginProbeAttempt("rel-1", relSpare)); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("a non-executor counted an attempt: %v", err)
+ }
+ // A replacement target has its own count.
+ o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "", relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if journalUint(o.Claim.Op, "probe_attempts."+relSpare) != 0 {
+ t.Fatal("the replacement inherited attempts")
+ }
+}
+
+func TestRelocationDischargeCompletesDiscoveryLikeARetirement(t *testing.T) {
+ ctx := context.Background()
+ s, o := relServing(t)
+ o = step(t, s, o, func(c *Claim) error {
+ c.Touched.DiscoveryOp = "disc"
+ c.Obligations = append(c.Obligations, Obligation{ID: "d1", Op: "disc", Kind: "park-local", Box: relSource, Retirement: "r", Owner: "box:" + relSource})
+ return nil
+ })
+ ref := putFence(t, s, relSource)
+ in := relStart(ReasonFailover, &ref)
+ in.Discharge = &LocalDischarge{Qualified: true}
+ o, err := StartRelocation(ctx, s, o, in, relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(o.Claim.Obligations) != 0 || !o.Claim.Touched.Complete || o.Claim.Touched.DiscoveryOp != "" {
+ t.Fatalf("discharging the final discovery chore: %+v touched %+v", o.Claim.Obligations, o.Claim.Touched)
+ }
+}
+
+func TestRelocationLossRetainsAFormerWritersSession(t *testing.T) {
+ // The source is fenced after the target's probe admission made the
+ // target the writer: the loss still names the source's last session.
+ ctx := context.Background()
+ s, o := relAtPhase(t, "probing+P")
+ before := o.Claim.Holder
+ var sourceAdmission AdmissionRecord
+ for _, a := range before.Admissions {
+ if a.Box == relSource {
+ sourceAdmission = a
+ }
+ }
+ o, err := FenceSource(ctx, s, o, "rel-1", putFence(t, s, relSource), nil, relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ l := o.Claim.EvidenceRefs[0]
+ if l.Box != relSource || l.Session != sourceAdmission.Session || l.Generation != sourceAdmission.Generation || l.LastAdmission == nil || !reflect.DeepEqual(*l.LastAdmission, sourceAdmission) ||
+ l.Incarnation != before.Incarnation || l.Stamp != before.Stamp {
+ t.Fatalf("retained: %+v, source admission %+v", l, sourceAdmission)
+ }
+ if err := o.Claim.Validate(); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestRelocationRetainsEveryFencedBoxAcrossCompletion(t *testing.T) {
+ // A drain whose target is terminated in installing: the replacement
+ // completes, and the dead target's loss is still on the claim.
+ ctx := context.Background()
+ s, o := relAtPhase(t, PhaseInstalling)
+ ref := putFence(t, s, relTarget)
+ o, err := ReplaceTarget(ctx, s, o, "rel-1", ref, nil, Target{relSpare, relEnrollment(relSpare)}, "P2", relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ o = step(t, s, o, ack(relSpare))
+ o = step(t, s, o, ToInstalled("rel-1", relPrepared("P2", 40, 7)))
+ o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim)))
+ o = step(t, s, o, ToProbing("rel-1", relObserved))
+ o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
+ Box: relSpare, Mode: "cold", NewGeneration: "P2", KeepStamp: true, Coverage: "released", Region: "ie", PredecessorRegion: "ie"}))
+ o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relSpare, Session: o.Claim.Holder.Session}))
+ o = step(t, s, o, CompleteRelocation("rel-1", counter()))
+ c := o.Claim
+ if len(c.EvidenceRefs) != 1 || c.EvidenceRefs[0].Box != relTarget || c.EvidenceRefs[0].Role != "target" || c.EvidenceRefs[0].Session != 0 || !slices.Contains(c.LossRefs, c.EvidenceRefs[0].ID) {
+ t.Fatalf("the abandoned target's loss: %+v refs %v", c.EvidenceRefs, c.LossRefs)
+ }
+ if err := c.Validate(); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestRelocationStartGuard(t *testing.T) {
+ serving := func(obs ...Obligation) *Claim {
+ return &Claim{Holder: &Holder{Incarnation: "i"}, Op: Op{Kind: KindServing, Exec: relSource}, Obligations: obs}
+ }
+ for _, tc := range []struct {
+ name string
+ ob Obligation
+ fenced bool
+ want error
+ }{
+ {"park-generation in the region (L11)", Obligation{Kind: "park-generation", Region: "ie"}, false, nil},
+ {"park-replica in the region", Obligation{Kind: "park-replica", Region: "ie"}, false, ErrPendingChore},
+ {"park-replica elsewhere", Obligation{Kind: "park-replica", Region: "uk"}, false, nil},
+ {"park-local on the target", Obligation{Kind: "park-local", Box: relTarget}, true, ErrPendingChore},
+ {"park-local on a live source", Obligation{Kind: "park-local", Box: relSource}, false, ErrPendingChore},
+ {"park-local on a fenced source", Obligation{Kind: "park-local", Box: relSource}, true, nil},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ if err := RelocationStartGuard(serving(tc.ob), "ie", relSource, relTarget, tc.fenced); !errors.Is(err, tc.want) {
+ t.Fatalf("got %v, want %v", err, tc.want)
+ }
+ })
+ }
+}
+
+func TestRelocationLocalInstallReceiptIsBound(t *testing.T) {
+ ctx := context.Background()
+ for _, tc := range []struct {
+ name string
+ mutate func(*LocalInstall)
+ }{
+ {"another box", func(l *LocalInstall) { l.Box = relSpare }},
+ {"another enrollment", func(l *LocalInstall) { l.Enrollment = relEnrollment(relSpare) }},
+ {"a later writer stamp", func(l *LocalInstall) { l.InstallStamp++ }},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ s, o := relAtPhase(t, PhaseInstalled)
+ l := localInstall(o.Claim)
+ tc.mutate(&l)
+ if _, err := Update(ctx, s, o, RecordLocalInstall("rel-1", l)); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("recorded: %v", err)
+ }
+ })
+ }
+ s, o := relAtPhase(t, PhaseInstalled)
+ l := localInstall(o.Claim)
+ o = step(t, s, o, RecordLocalInstall("rel-1", l))
+ if _, err := Update(ctx, s, o, RecordLocalInstall("rel-1", l)); !errors.Is(err, ErrAlreadyRecorded) {
+ t.Fatalf("replay: %v", err)
+ }
+ if _, err := Update(ctx, s, o, MarkStuck("rel-1", "operator")); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("stuck between installed and a probe: %v", err)
+ }
+ // B1-2: probing is refused on another box's receipt, even one in the journal.
+ foreign := l
+ foreign.Box = relSpare
+ raw, _ := json.Marshal(foreign)
+ o.Claim.Op.Journal["local_install"] = string(raw)
+ if _, err := Update(ctx, s, o, ToProbing("rel-1", relObserved)); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("probing on another box's receipt: %v", err)
+ }
+}
+
+// Round 5 finding 1: a spare that still owes a local retirement for the
+// host is never named a replacement target, whatever phase the
+// replacement resumes at (installing and installed hand it op.exec in the
+// same CAS); a clean spare is.
+func TestRelocationReplacementRefusesABoxOwingALocalRetirement(t *testing.T) {
+ ctx := context.Background()
+ for _, phase := range []string{PhaseStarting, PhaseInstalling, PhaseInstalled} {
+ t.Run(phase, func(t *testing.T) {
+ s, o := relAtPhase(t, phase)
+ o = step(t, s, o, func(c *Claim) error {
+ c.Obligations = append(c.Obligations, Obligation{ID: "old", Op: "old-op", Kind: "park-local", Box: relSpare, Retirement: "r", Owner: "box:" + relSpare})
+ return nil
+ })
+ ref := putFence(t, s, relTarget)
+ if _, err := ReplaceTarget(ctx, s, o, "rel-1", ref, nil, Target{relSpare, relEnrollment(relSpare)}, "P2", relAuthority{}); !errors.Is(err, ErrPendingChore) {
+ t.Fatalf("a box owing park-local was named: %v", err)
+ }
+ if _, err := Update(ctx, s, o, NameTarget("rel-1", Target{relSpare, relEnrollment(relSpare)})); err == nil {
+ t.Fatal("NameTarget named a box owing park-local")
+ }
+ o, err := ReplaceTarget(ctx, s, o, "rel-1", ref, nil, Target{relSpare2, relEnrollment(relSpare2)}, "P2", relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ c := o.Claim
+ if c.Op.Intent["target_box"] != relSpare2 || slices.Contains(c.Op.Participants, relSpare) || slices.Contains(c.Touched.Boxes, relSpare) {
+ t.Fatalf("the clean spare was not chosen: %+v touched %v", c.Op, c.Touched.Boxes)
+ }
+ if phase != PhaseStarting && c.Op.Exec != relSpare2 {
+ t.Fatalf("exec %s", c.Op.Exec)
+ }
+ })
+ }
+}
+
+// Round 5 finding 2: the automatic fence passes the dead box's regional
+// work (park-generation, park-replica owned by box:<b>) to its region in
+// the same CAS, as DeclareLost does for a declared loss, keeping every
+// other field; another box's work is untouched; local discharge, qualified
+// or not, changes nothing about it.
+func TestRelocationFenceReassignsTheDeadBoxsRegionalWork(t *testing.T) {
+ ctx := context.Background()
+ gen := Obligation{ID: "pg", Op: "old-op", Kind: "park-generation", Region: "ie", Generation: "g0", Retirement: "rg", Owner: "box:" + relSource}
+ rep := Obligation{ID: "pr", Op: "old-op", Kind: "park-replica", Region: "uk", Retirement: "rr", Owner: "box:" + relSource}
+ other := Obligation{ID: "po", Op: "old-op", Kind: "park-generation", Region: "ie", Generation: "g0b", Retirement: "ro", Owner: "box:" + relSpare}
+ local := Obligation{ID: "pl", Op: "old-op", Kind: "park-local", Box: relSource, Retirement: "rl", Owner: "box:" + relSource}
+ for _, qualified := range []bool{false, true} {
+ s, o := relServing(t)
+ o = step(t, s, o, func(c *Claim) error {
+ c.Obligations = append(c.Obligations, gen, rep, other, local)
+ return nil
+ })
+ ref := putFence(t, s, relSource)
+ in := relStart(ReasonFailover, &ref)
+ in.Discharge = &LocalDischarge{Qualified: qualified, Records: []string{"control/box-volumes/x/1.json"}}
+ o, err := StartRelocation(ctx, s, o, in, relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ byID := map[string]Obligation{}
+ for _, ob := range o.Claim.Obligations {
+ byID[ob.ID] = ob
+ }
+ wantGen, wantRep := gen, rep
+ wantGen.Owner, wantRep.Owner = "region:ie", "region:uk"
+ if !reflect.DeepEqual(byID["pg"], wantGen) || !reflect.DeepEqual(byID["pr"], wantRep) || !reflect.DeepEqual(byID["po"], other) {
+ t.Fatalf("qualified=%v: regional work %+v", qualified, o.Claim.Obligations)
+ }
+ if _, owes := byID["pl"]; owes == qualified {
+ t.Fatalf("qualified=%v: the local chore %+v", qualified, o.Claim.Obligations)
+ }
+ want := []ObligationRef{{ID: "pg", Op: "old-op", Retirement: "rg"}, {ID: "pr", Op: "old-op", Retirement: "rr"}}
+ if len(o.Claim.EvidenceRefs) != 1 || !reflect.DeepEqual(o.Claim.EvidenceRefs[0].Reassigned, want) {
+ t.Fatalf("qualified=%v: the retained loss does not name the reassignment: %+v", qualified, o.Claim.EvidenceRefs)
+ }
+ if err := o.Claim.Validate(); err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
+// Round 11's sweep: P's opening is journaled with the admission that made
+// it, so it survives AdmitWriter's trim of holder.admissions to eight. The
+// removal below stands in for that trim (generation.go drops a covered,
+// loss-archived head once more than eight are kept); with P's admission
+// gone from the list, P is still opened: never reopened, and the local
+// install is never remade.
+func TestRelocationPOpenedOutlivesTheAdmissionTrim(t *testing.T) {
+ ctx := context.Background()
+ s, o := relAtPhase(t, "probing+P")
+ if o.Claim.Op.Journal["prepared_opened"] != "P1" {
+ t.Fatalf("the opening was not journaled: %+v", o.Claim.Op.Journal)
+ }
+ o = step(t, s, o, func(c *Claim) error {
+ c.Holder.Admissions = slices.DeleteFunc(c.Holder.Admissions, func(a AdmissionRecord) bool { return a.Generation == "P1" })
+ return nil
+ })
+ if !PreparedOpened(o.Claim) {
+ t.Fatal("P forgotten with its admission")
+ }
+ again := Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
+ Box: relTarget, Mode: "cold", NewGeneration: "P1", KeepStamp: true, Region: "ie", PredecessorRegion: "ie"}
+ if _, err := Update(ctx, s, o, AdmitWriter(again)); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("P reopened after the trim: %v", err)
+ }
+ if _, err := Update(ctx, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim))); err == nil {
+ t.Fatal("the local install was remade after P was opened")
+ }
+}
+
+// Round 14 finding 1: P's prepared manifest is at sequence 0 whatever the
+// source's; the pin's sequence (7 here) is its provenance. ToInstalled
+// refuses a prepared manifest claimed at the source's sequence, and one
+// built from another source revision, and binds the one it accepts to
+// the installation stamp.
+func TestRelocationPreparedManifestIsSequenceZero(t *testing.T) {
+ ctx := context.Background()
+ s, o := relAtPhase(t, PhaseInstalling)
+ var pin RecoveryPin
+ readJournal(o.Claim.Op, "pin", &pin)
+ if pin.FilesSeq != 7 {
+ t.Fatalf("pin at %d", pin.FilesSeq)
+ }
+ atSource := relPrepared("P1", 40, 7)
+ atSource.FilesSeq = 7
+ if _, err := Update(ctx, s, o, ToInstalled("rel-1", atSource)); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("a prepared manifest at the source's sequence: %v", err)
+ }
+ if _, err := Update(ctx, s, o, ToInstalled("rel-1", relPrepared("P1", 40, 6))); !errors.Is(err, ErrRelocation) {
+ t.Fatalf("a prepared manifest from another source revision: %v", err)
+ }
+ o = step(t, s, o, ToInstalled("rel-1", relPrepared("P1", 40, 7)))
+ var a PreparedAuthority
+ readJournal(o.Claim.Op, "prepared_authority", &a)
+ if a.FilesSeq != 0 || a.SourceFilesSeq != 7 || a.InstallStamp == 0 || a.InstallStamp != journalUint(o.Claim.Op, "install_stamp") {
+ t.Fatalf("journaled authority: %+v", a)
+ }
+}
+
+// Two-box model finding F3: a target replaced in installing leaves its P
+// unopened; the replacement builds P'. The serving CAS queues a
+// park-generation for P in the region, and none for the opened P'.
+func TestRelocationParksEveryAbandonedPreparedGeneration(t *testing.T) {
+ ctx := context.Background()
+ s, o := relAtPhase(t, PhaseInstalling)
+ o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "P2", relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ var abandoned []string
+ if !readJournal(o.Claim.Op, "abandoned_prepared", &abandoned) || !slices.Equal(abandoned, []string{"P1"}) {
+ t.Fatalf("abandoned: %v", abandoned)
+ }
+ o = step(t, s, o, ack(relSpare))
+ o = step(t, s, o, ToInstalled("rel-1", relPrepared("P2", 40, 7)))
+ o = step(t, s, o, RecordLocalInstall("rel-1", localInstall(o.Claim)))
+ o = step(t, s, o, ToProbing("rel-1", relObserved))
+ o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
+ Box: relSpare, Mode: "cold", NewGeneration: "P2", KeepStamp: true, Coverage: "released", Region: "ie", PredecessorRegion: "ie"}))
+ o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relSpare, Session: o.Claim.Holder.Session}))
+ o = step(t, s, o, CompleteRelocation("rel-1", counter()))
+ var parks []string
+ for _, ob := range o.Claim.Obligations {
+ if ob.Kind == "park-generation" && ob.Region == "ie" && ob.Owner == "region:ie" {
+ parks = append(parks, ob.Generation)
+ }
+ }
+ if !slices.Contains(parks, "P1") || slices.Contains(parks, "P2") {
+ t.Fatalf("park-generation chores: %v (%+v)", parks, o.Claim.Obligations)
+ }
+ if err := o.Claim.Validate(); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// Round 17 finding 3: the target's probe deadline passes before any
+// admission (P1 unopened), it releases in probe-failed, then it is
+// terminated; the replacement's default branch drops P1. It is recorded
+// first, so the completion of the nested relocation parks it.
+func TestRelocationParksAPDroppedFromProbeFailed(t *testing.T) {
+ ctx := context.Background()
+ s, o := relAtPhase(t, PhaseProbing)
+ if PreparedOpened(o.Claim) {
+ t.Fatal("P1 was opened")
+ }
+ o = step(t, s, o, ToProbeFailed("rel-1", relObserved))
+ o, err := ReplaceTarget(ctx, s, o, "rel-1", putFence(t, s, relTarget), nil, Target{relSpare, relEnrollment(relSpare)}, "", relAuthority{})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if o.Claim.Op.Phase != PhasePinning || o.Claim.Op.Journal["prepared"] != "" {
+ t.Fatalf("nested: %+v", o.Claim.Op)
+ }
+ var abandoned []string
+ if !readJournal(o.Claim.Op, "abandoned_prepared", &abandoned) || !slices.Equal(abandoned, []string{"P1"}) {
+ t.Fatalf("abandoned: %v", abandoned)
+ }
+ o = step(t, s, o, ack(relSpare))
+ o = step(t, s, o, JournalPin("rel-1", relPin(o, 40, 7, relObserved.Add(FailoverPinDelay))))
+ o = toProbing(t, s, o, "P2", 40)
+ o = step(t, s, o, AdmitWriter(Admission{ObservedOpID: "rel-1", ObservedPhase: PhaseProbing, ObservedSession: o.Claim.Holder.Session, ObservedGeneration: o.Claim.Holder.Generation,
+ Box: relSpare, Mode: "cold", NewGeneration: "P2", KeepStamp: true, Region: "ie", PredecessorRegion: "ie"}))
+ o = step(t, s, o, RecordProbe("rel-1", Probe{Box: relSpare, Session: o.Claim.Holder.Session}))
+ o = step(t, s, o, CompleteRelocation("rel-1", counter()))
+ parked := false
+ for _, ob := range o.Claim.Obligations {
+ if ob.Kind == "park-generation" && ob.Generation == "P1" && ob.Region == "ie" {
+ parked = true
+ }
+ }
+ if !parked {
+ t.Fatalf("P1 was not parked: %+v", o.Claim.Obligations)
+ }
+}