amadan

carlos / platform Public

hostclaim: RollBack, TargetReleased, ReturnToSource, ReleaseMove, rev_phase (§7.8, K4, K8)

The claim half of the rollback and the return, per Task 2 and its rulings:

- RollBack: landing before the commit to restoring/releasing, rollback id
  in the intent, the target's stamp F journalled (J15). Races Commit.
- RollBackFlipped: writes the flip first over moving/sealed (J16).
- TargetReleased: exec to none on evidence_target, released, the target
  region's location park and CoverRegionPredecessors in one write; the
  target's park-local only when its inventory lists F (J15).
- ReturnToSource: claimed with holder.stamp = source_stamp and, clean,
  holder.generation = src_gen; admissions untouched. Authority by
  evidence_target, a loss.<id> declaration, or a fence the caller
  verified with VerifyReturnFence before Update (J5).
- SetRevPhase (evidence/verify by baseline, J8) and SetRestoringPhase
  mirror probing, probed and opened onto op.phase (J1);
  RecoveryPhaseOrder names both orders, tested against WakePhase.
- QueueReturnLanePark: moves/<host>/<rb id> in the source's region at
  probed, once.
- ReleaseMove also releases landing/committed, restoring/opened and
  reverting/opened (by op.phase, exec checked); moving/"" and
  moving/aborting and the released_move journal are kept (J6).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 3d6a233013bd7a2aaf8451abc7601d5465d157da parent fe7dc8a
3 files changed, +1231 −16
  • internal/hostclaim/move.go +395 −16
  • internal/hostclaim/move_recovery_external_test.go +69 −0
  • internal/hostclaim/move_test.go +767 −0
diff --git a/internal/hostclaim/move.go b/internal/hostclaim/move.go
index c13d31df1..4215bfcec 100644
--- a/internal/hostclaim/move.go
+++ b/internal/hostclaim/move.go
@@ -1,10 +1,14 @@
package hostclaim
import (
+ "context"
"fmt"
+ "regexp"
"strconv"
"strings"
"time"
+
+ "github.com/carlosframework/platform/internal/release"
)
// Move phases on the claim's op (moves spec §7.5, §19.1). The record
@@ -128,22 +132,31 @@ func RequireMoveOp(opID, kind, phase string) func(*Claim) error {
}
}
-// ReleaseMove is K8: the interim abort hands the host back to serving. It
-// is legal only from moving, before the seal (phase "" or "aborting"), and
-// only once no obligation under this op remains. The serving op keeps
-// exec and records which move released it: operation-bound completion
+// ReleaseMove is K8: a move hands the host back to serving. It is legal
+// from the interim abort (moving before the seal, phase "" or "aborting"),
+// and from the end of each move outcome, read from op.phase (J1, J6): a
+// committed landing, an opened rollback (restoring) and an opened revert
+// (reverting). In those three the executor named must be the one executing.
+// It is legal only once no obligation under this op remains. The serving op
+// keeps exec and records which move released it: operation-bound completion
// evidence for the abort's crash boundary (RequireReleasedMove).
func ReleaseMove(opID, execBox string) func(*Claim) error {
return func(c *Claim) error {
if err := RequireOp(opID)(c); err != nil {
return err
}
- if c.Op.Kind != KindMoving || (c.Op.Phase != PhaseMoving && c.Op.Phase != PhaseAborting) {
- return fmt.Errorf("%w: op is %s/%q, a release needs moving before the seal", ErrWrongState, c.Op.Kind, c.Op.Phase)
+ abort := c.Op.Kind == KindMoving && (c.Op.Phase == PhaseMoving || c.Op.Phase == PhaseAborting)
+ ended := (c.Op.Kind == KindLanding && c.Op.Phase == PhaseCommitted) ||
+ ((c.Op.Kind == KindRestoring || c.Op.Kind == KindReverting) && c.Op.Phase == PhaseOpened)
+ if !abort && !ended {
+ return fmt.Errorf("%w: op is %s/%q, a release needs moving before the seal, a committed landing or an opened recovery", ErrWrongState, c.Op.Kind, c.Op.Phase)
}
if execBox == "" {
return fmt.Errorf("%w: a release names the executor", ErrMalformed)
}
+ if ended && c.Op.Exec != execBox {
+ return fmt.Errorf("%w: exec is %q, the release names %q", ErrWrongState, c.Op.Exec, execBox)
+ }
for _, o := range c.Obligations {
if o.Op == opID {
return fmt.Errorf("%w: %s %s%s is still owed under this move", ErrPendingChore, o.Kind, o.Region, o.Box)
@@ -296,7 +309,7 @@ func Revert(opID, sourceBox, targetRegion string, stagingScope *ParkScope, newID
}
if c.Op.Kind == KindReverting {
if c.Op.Exec == sourceBox && sourceBox != "" && len(c.Op.Participants) > 0 && c.Op.Participants[0] == sourceBox &&
- c.Op.Journal[JournalEvidenceSource] != "" && c.Op.Journal[JournalRevPhase] == "evidence" &&
+ c.Op.Journal[JournalEvidenceSource] != "" && c.Op.Journal[JournalRevPhase] == RevPhaseEvidence &&
targetRegion != "" && c.Op.Intent["to"] == targetRegion && stagingScope != nil {
for _, o := range c.Obligations {
if o.Op == opID && o.Kind == "park-replica" && o.Region == targetRegion && o.Park != nil && sameParkScope(*o.Park, *stagingScope) {
@@ -326,7 +339,7 @@ func Revert(opID, sourceBox, targetRegion string, stagingScope *ParkScope, newID
}
c.Op.Kind, c.Op.Exec = KindReverting, sourceBox
if c.Op.Journal[JournalRevPhase] == "" {
- setJournal(c, JournalRevPhase, "evidence")
+ setJournal(c, JournalRevPhase, RevPhaseEvidence)
}
for _, o := range c.Obligations {
if o.Op == opID && o.Kind == "park-replica" && o.Region == targetRegion {
@@ -418,13 +431,9 @@ func queueLocalPark(c *Claim, opID, box string, newID func() string) {
c.Obligations = append(c.Obligations, Obligation{ID: newID(), Op: opID, Kind: "park-local", Box: box, Retirement: newID(), Owner: "box:" + box})
}
-// Claim states in which a region's whole location is retired (§6.4, §7.8):
-// the landing after the commit, and a rollback from the target's release.
-const (
- PhaseCommitted = "committed"
- phaseRestoringReleased = "released"
- phaseRestoringClaimed = "claimed"
-)
+// PhaseCommitted is the landing after the commit: the source region's whole
+// location is retired from here (§6.4, §7.9).
+const PhaseCommitted = "committed"
// requireRetiredRegion is QueueLocationPark's lifecycle check. A whole
// location is parked only where the region has stopped being authoritative:
@@ -445,7 +454,7 @@ func requireRetiredRegion(c *Claim, region, box string) error {
if to != "" && region != to && (box == "" || box == c.Op.Intent["source"]) {
return nil
}
- case c.Op.Kind == KindRestoring && (c.Op.Phase == phaseRestoringReleased || c.Op.Phase == phaseRestoringClaimed):
+ case c.Op.Kind == KindRestoring && (c.Op.Phase == PhaseReleased || c.Op.Phase == PhaseClaimed):
if to != "" && region == to && (box == "" || box == c.Op.Intent["target"]) {
return nil
}
@@ -657,3 +666,373 @@ func QueueResidueParks(opID, region string, location ParkScope, generations []st
return nil
}
}
+
+// Recovery phases on the claim's op (spec §7.4, §7.8, §7.14; J1). The record
+// carries the detailed recovery journal; the claim mirrors the phases the
+// wake policy and the release key on, as the landing mirrors its own.
+//
+// restoring (a rollback, after the flip): the target's releasing and
+// released, the source's claimed, then probing (written before the source's
+// first recovery attempt), probed and opened.
+//
+// reverting (before the flip): op.phase stays the moving phase the revert
+// began from ("" or "sealed") through rev_phase evidence and verify, then
+// mirrors probing, probed and opened in the CAS that writes rev_phase.
+const (
+ PhaseReleasing = "releasing" // restoring: the target owes its stop evidence
+ PhaseReleased = "released" // restoring: the target gave execution up
+ PhaseClaimed = "claimed" // restoring: the source took execution back
+ PhaseProbing = "probing" // restoring, reverting: the recovery probe
+ PhaseProbed = "probed" // restoring, reverting: the probe passed
+ PhaseOpened = "opened" // restoring, reverting: the source serves again
+
+ RevPhaseEvidence = "evidence" // rev_phase: stop evidence before anything starts
+ RevPhaseVerify = "verify" // rev_phase: files and timers against the stopped digests
+)
+
+// JournalTargetStamp is the target installation's stamp F, journalled at the
+// rollback for its release (J15): the target's park-local is queued only
+// when its inventory lists paths of F.
+const JournalTargetStamp = "target_stamp"
+
+// journalReturnLane marks the return lane's park as queued, so a replay after
+// the park engine has completed it does not queue it again.
+const journalReturnLane = "return_lane"
+
+// RecoveryPhaseOrder is the op.phase order a recovery kind writes, or nil for
+// any other kind. A revert's phases before probing are the moving phase it
+// began from and are not listed.
+func RecoveryPhaseOrder(kind string) []string {
+ switch kind {
+ case KindRestoring:
+ return []string{PhaseReleasing, PhaseReleased, PhaseClaimed, PhaseProbing, PhaseProbed, PhaseOpened}
+ case KindReverting:
+ return []string{PhaseProbing, PhaseProbed, PhaseOpened}
+ }
+ return nil
+}
+
+// rollbackIDShape restates moves' rb_ op id shape (moves imports hostclaim, so
+// it cannot be called from here). TestRollBackTakesMintedRollbackIDs holds
+// the two together.
+var rollbackIDShape = regexp.MustCompile(`^rb_[a-z2-7]{26}$`)
+
+// inOneWrite runs steps in order as one transition: if any refuses, the claim
+// is put back exactly as it was found.
+func inOneWrite(c *Claim, steps ...func(*Claim) error) error {
+ saved, err := cloneClaim(c)
+ if err != nil {
+ return err
+ }
+ for _, step := range steps {
+ if err := step(c); err != nil {
+ *c = *saved
+ return err
+ }
+ }
+ return nil
+}
+
+// RollBack is the claim half of the record's rollback CAS (§7.5 rolling_back,
+// §7.8): landing, at any phase before the commit, to restoring/releasing
+// under the same op id, with the rollback id in the intent. Execution stays
+// with the target, which still owes its stop evidence; the holder is not
+// touched. The target installation's stamp F (holder.stamp now) is
+// journalled for TargetReleased (J15).
+//
+// The commit and the rollback race on the claim as on the record: a committed
+// landing refuses the rollback, and Commit refuses a restoring op. A replay
+// that finds this rollback already written (any restoring phase) is a no-op;
+// another rollback id is refused.
+func RollBack(opID, rbID string) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := RequireOp(opID)(c); err != nil {
+ return err
+ }
+ if !rollbackIDShape.MatchString(rbID) {
+ return fmt.Errorf("%w: %q is not a rollback id", ErrMalformed, rbID)
+ }
+ if c.Op.Kind == KindRestoring {
+ if c.Op.Intent["rollback"] == rbID {
+ return nil
+ }
+ return fmt.Errorf("%w: the op is restoring under rollback %q", ErrWrongState, c.Op.Intent["rollback"])
+ }
+ if c.Op.Kind != KindLanding || c.Op.Phase == PhaseCommitted || c.Holder == nil {
+ return fmt.Errorf("%w: op is %s/%q, a rollback needs a landing before the commit", ErrWrongState, c.Op.Kind, c.Op.Phase)
+ }
+ intent := make(map[string]string, len(c.Op.Intent)+1)
+ for k, v := range c.Op.Intent {
+ intent[k] = v
+ }
+ intent["rollback"] = rbID
+ c.Op.Kind, c.Op.Phase, c.Op.Intent = KindRestoring, PhaseReleasing, intent
+ setJournal(c, JournalTargetStamp, strconv.FormatUint(c.Holder.Stamp, 10))
+ return nil
+ }
+}
+
+// RollBackFlipped is the rollback's claim writer for a record already at
+// landing (J16): when the claim is still moving/sealed (the flip's claim half
+// not yet written, G12) it writes Flip first, then RollBack, in one write;
+// over any other kind it is RollBack. A refused flip leaves the claim as found.
+func RollBackFlipped(opID, rbID, targetBox string, sourceStamp uint64, srcGen string) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := RequireOp(opID)(c); err != nil {
+ return err
+ }
+ if !rollbackIDShape.MatchString(rbID) {
+ return fmt.Errorf("%w: %q is not a rollback id", ErrMalformed, rbID)
+ }
+ if c.Op.Kind != KindMoving {
+ return RollBack(opID, rbID)(c)
+ }
+ return inOneWrite(c, Flip(opID, targetBox, sourceStamp, srcGen), RollBack(opID, rbID))
+ }
+}
+
+// TargetReleased is the target's released CAS (§7.8): restoring/releasing,
+// with the target executing, gives execution up citing its stop evidence
+// (ExecToNone, evidence_target), moves to released and, in the same write,
+// queues the park of the target region's whole location (QueueLocationPark,
+// D6) and covers every admission record whose predecessor is in the target
+// region (CoverRegionPredecessors, claim rev 6.14 item 6, R12-3). The
+// target's park-local is queued only when its inventory lists paths of the
+// stamp journalled at the rollback (J15); otherwise none is, and the
+// target's cleaning parks its unregistered leftovers itself.
+//
+// A replay that finds the release written with the same evidence (released
+// or any later phase) is a no-op and queues nothing, even after a chore has
+// completed. A refused release leaves the claim as found.
+func TargetReleased(opID, targetBox, targetRegion, evidenceKey string, scope ParkScope, newID func() string) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := RequireOp(opID)(c); err != nil {
+ return err
+ }
+ if c.Op.Kind != KindRestoring {
+ return fmt.Errorf("%w: op is %s/%q, a release needs a rollback", ErrWrongState, c.Op.Kind, c.Op.Phase)
+ }
+ if targetBox == "" || targetRegion == "" || evidenceKey == "" || newID == nil {
+ return fmt.Errorf("%w: a release names the target, its region and its evidence", ErrMalformed)
+ }
+ if c.Op.Phase != PhaseReleasing {
+ switch c.Op.Phase {
+ case PhaseReleased, PhaseClaimed, PhaseProbing, PhaseProbed, PhaseOpened:
+ if c.Op.Journal[JournalEvidenceTarget] == evidenceKey {
+ return nil
+ }
+ }
+ return fmt.Errorf("%w: op is %s/%q, not releasing with this evidence", ErrWrongState, c.Op.Kind, c.Op.Phase)
+ }
+ if len(c.Op.Participants) != 2 || c.Op.Participants[1] != targetBox || c.Op.Intent["target"] != targetBox || c.Op.Exec != targetBox {
+ return fmt.Errorf("%w: %q is not the executing target", ErrWrongState, targetBox)
+ }
+ if c.Op.Intent["to"] != targetRegion {
+ return fmt.Errorf("%w: %q is not the move's target region", ErrWrongState, targetRegion)
+ }
+ parkBox := ""
+ if stamp, err := strconv.ParseUint(c.Op.Journal[JournalTargetStamp], 10, 64); err == nil && InventoryHasStamp(c, targetBox, stamp) {
+ parkBox = targetBox
+ }
+ return inOneWrite(c,
+ ExecToNone(opID, targetBox, evidenceKey, JournalEvidenceTarget),
+ func(c *Claim) error { c.Op.Phase = PhaseReleased; return nil },
+ QueueLocationPark(opID, targetRegion, parkBox, scope, newID),
+ CoverRegionPredecessors(opID, PhaseReleased, targetRegion),
+ )
+ }
+}
+
+// ReturnFence is a termination fence for a move's target that the caller
+// verified before Update (VerifyReturnFence, J5): the op, the box and the
+// exact fence.<box> journal entry it verified. The zero value vouches for
+// nothing.
+type ReturnFence struct{ op, box, ref string }
+
+// VerifyReturnFence re-reads and verifies the fence the op journals for box
+// (VerifyOperationFence) and returns what ReturnToSource checks inside its
+// CAS. Run it on the observation that Update will write over.
+func VerifyReturnFence(ctx context.Context, s release.Store, authority FenceAuthority, op Op, box string) (ReturnFence, error) {
+ if err := VerifyOperationFence(ctx, s, authority, op, box); err != nil {
+ return ReturnFence{}, err
+ }
+ return ReturnFence{op: op.ID, box: box, ref: op.Journal["fence."+box]}, nil
+}
+
+// targetGaveUp is ReturnToSource's authority: the target's own stop evidence
+// (evidence_target); or an operator's declaration under this op (a loss.<id>
+// journal entry the claim records in loss_refs) that took the target out of
+// the participants; or a verified termination fence whose journal entry the
+// claim still carries.
+func targetGaveUp(c *Claim, opID string, fence ReturnFence) bool {
+ target := c.Op.Intent["target"]
+ if target == "" {
+ return false
+ }
+ if c.Op.Journal[JournalEvidenceTarget] != "" {
+ return true
+ }
+ if !stringPresent(c.Op.Participants, target) {
+ for _, id := range c.LossRefs {
+ if _, ok := c.Op.Journal["loss."+id]; ok {
+ return true
+ }
+ }
+ }
+ return fence.op == opID && fence.box == target && fence.ref != "" && c.Op.Journal["fence."+target] == fence.ref
+}
+
+// ReturnToSource is the source's claimed CAS (§7.8, K4): restoring/released
+// with nobody executing ("" after the target's evidence, "none" after a
+// declaration) and the target's authority given up (targetGaveUp) makes the
+// source, the move's first participant, the executor again at claimed.
+// holder.stamp is the source installation's stamp S the flip journalled; a
+// clean return also names the source's generation (src_gen) again. The
+// admissions are not touched (R11-4): the source's re-admission names the
+// target's writer, their last record, as its predecessor.
+//
+// A declared-lost target is not released by TargetReleased: the resolve door
+// steps the claim to released and queues the target region's park itself
+// (J16), so this transition is the same for every form. A replay at claimed
+// with the source executing is a no-op.
+func ReturnToSource(opID, sourceBox string, clean bool, fence ReturnFence) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := RequireOp(opID)(c); err != nil {
+ return err
+ }
+ if sourceBox == "" {
+ return fmt.Errorf("%w: a return names the source", ErrMalformed)
+ }
+ if c.Op.Kind != KindRestoring || c.Holder == nil {
+ return fmt.Errorf("%w: op is %s/%q, a return needs a rollback", ErrWrongState, c.Op.Kind, c.Op.Phase)
+ }
+ if c.Op.Phase == PhaseClaimed && c.Op.Exec == sourceBox {
+ return nil
+ }
+ if c.Op.Phase != PhaseReleased || (c.Op.Exec != "" && c.Op.Exec != "none") {
+ return fmt.Errorf("%w: op is %s/%q with exec %q, a return needs the target released", ErrWrongState, c.Op.Kind, c.Op.Phase, c.Op.Exec)
+ }
+ if len(c.Op.Participants) == 0 || c.Op.Participants[0] != sourceBox || c.Op.Intent["source"] != sourceBox {
+ return fmt.Errorf("%w: %q is not the move's source", ErrWrongState, sourceBox)
+ }
+ if !targetGaveUp(c, opID, fence) {
+ return fmt.Errorf("%w: no evidence, declaration or verified fence for the target", ErrWrongState)
+ }
+ stamp, err := strconv.ParseUint(c.Op.Journal[JournalSourceStamp], 10, 64)
+ srcGen, flipped := c.Op.Journal[JournalSrcGen]
+ if err != nil || stamp == 0 || !flipped {
+ return fmt.Errorf("%w: the flip's source stamp and generation are not journalled", ErrWrongState)
+ }
+ c.Op.Exec, c.Op.Phase = sourceBox, PhaseClaimed
+ c.Holder.Stamp = stamp
+ if clean {
+ c.Holder.Generation = srcGen
+ }
+ return nil
+ }
+}
+
+// SetRestoringPhase steps a rollback's source recovery on the claim (J1):
+// claimed, probing, probed, opened, one step at a time, with the source
+// executing. releasing to released is TargetReleased's and released to
+// claimed is ReturnToSource's. A replay finds the phase moved and is refused.
+func SetRestoringPhase(opID, from, to string) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := RequireMoveOp(opID, KindRestoring, from)(c); err != nil {
+ return err
+ }
+ step := from + ">" + to
+ if step != PhaseClaimed+">"+PhaseProbing && step != PhaseProbing+">"+PhaseProbed && step != PhaseProbed+">"+PhaseOpened {
+ return fmt.Errorf("%w: restoring does not step %q to %q", ErrWrongState, from, to)
+ }
+ if len(c.Op.Participants) == 0 || c.Op.Exec != c.Op.Participants[0] {
+ return fmt.Errorf("%w: exec %q is not the source", ErrWrongState, c.Op.Exec)
+ }
+ c.Op.Phase = to
+ return nil
+ }
+}
+
+// SetRevPhase steps a revert's journalled rev_phase (§7.4): evidence, verify,
+// probing, probed, opened. verify needs the files baseline the record keeps
+// from stopped (sourceBaseline); a revert from stopping has none and goes from
+// evidence straight to probing (J8). From probing on, op.phase is written in
+// the same CAS (J1); before it, op.phase is the moving phase the revert began
+// from. The source must be executing. A replay finds the phase moved and is
+// refused.
+func SetRevPhase(opID, from, to string, sourceBaseline bool) func(*Claim) error {
+ return func(c *Claim) error {
+ if err := RequireOp(opID)(c); err != nil {
+ return err
+ }
+ if c.Op.Kind != KindReverting || c.Op.Journal[JournalRevPhase] != from {
+ return fmt.Errorf("%w: op is %s with rev_phase %q, the step expects reverting at %q", ErrWrongState, c.Op.Kind, c.Op.Journal[JournalRevPhase], from)
+ }
+ if len(c.Op.Participants) == 0 || c.Op.Exec != c.Op.Participants[0] {
+ return fmt.Errorf("%w: exec %q is not the source", ErrWrongState, c.Op.Exec)
+ }
+ legal := false
+ switch from + ">" + to {
+ case RevPhaseEvidence + ">" + RevPhaseVerify:
+ legal = sourceBaseline
+ case RevPhaseEvidence + ">" + PhaseProbing:
+ legal = !sourceBaseline
+ case RevPhaseVerify + ">" + PhaseProbing, PhaseProbing + ">" + PhaseProbed, PhaseProbed + ">" + PhaseOpened:
+ legal = true
+ }
+ if !legal {
+ return fmt.Errorf("%w: rev_phase does not step %q to %q (baseline %v)", ErrWrongState, from, to, sourceBaseline)
+ }
+ mirrored := c.Op.Phase == from
+ if from == RevPhaseEvidence || from == RevPhaseVerify {
+ mirrored = c.Op.Phase == PhaseMoving || c.Op.Phase == PhaseSealed
+ }
+ if !mirrored {
+ return fmt.Errorf("%w: op.phase %q disagrees with rev_phase %q", ErrWrongState, c.Op.Phase, from)
+ }
+ setJournal(c, JournalRevPhase, to)
+ if to != RevPhaseVerify {
+ c.Op.Phase = to
+ }
+ return nil
+ }
+}
+
+// QueueReturnLanePark queues the park of a dirty rollback's return lane,
+// moves/<host>/<rollback id>, in the source's region (§7.8 probed): the
+// rollback's own lane, never the move's, the live location or the host-wide
+// staging prefix. It is legal at restoring/probed, composed into the probed
+// CAS. Idempotent: a chore already owed is not queued twice, and once the
+// park engine has completed it a replay does not queue it again.
+func QueueReturnLanePark(opID, sourceRegion string, scope ParkScope, newID func() string) func(*Claim) error {
+ return func(c *Claim) error {
+ var err error
+ if scope, err = parkQueueGuard(c, opID, sourceRegion, scope, newID); err != nil {
+ return err
+ }
+ rb := c.Op.Intent["rollback"]
+ if c.Op.Kind != KindRestoring || c.Op.Phase != PhaseProbed || !rollbackIDShape.MatchString(rb) {
+ return fmt.Errorf("%w: op is %s/%q, the return lane is parked at a rollback's probed", ErrWrongState, c.Op.Kind, c.Op.Phase)
+ }
+ if sourceRegion == c.Op.Intent["to"] {
+ return fmt.Errorf("%w: the return lane is in the source's region, not %s", ErrWrongState, sourceRegion)
+ }
+ if scope.Location != "moves/"+c.Host+"/"+rb {
+ return fmt.Errorf("%w: the return lane is this rollback's own lane, moves/%s/%s", ErrMalformed, c.Host, rb)
+ }
+ owed := false
+ for _, o := range c.Obligations {
+ if o.Op == opID && o.Kind == "park-replica" && o.Region == sourceRegion && o.Park != nil && isStagingLocation(o.Park.Location) {
+ owed = true
+ }
+ }
+ if !owed && c.Op.Journal[journalReturnLane] == scope.Location {
+ return nil
+ }
+ if _, err := queueReplicaPark(c, opID, sourceRegion, scope, newID); err != nil {
+ return err
+ }
+ setJournal(c, journalReturnLane, scope.Location)
+ return nil
+ }
+}
diff --git a/internal/hostclaim/move_recovery_external_test.go b/internal/hostclaim/move_recovery_external_test.go
new file mode 100644
index 000000000..956d4598c
--- /dev/null
+++ b/internal/hostclaim/move_recovery_external_test.go
@@ -0,0 +1,69 @@
+package hostclaim_test
+
+import (
+ "crypto/rand"
+ "errors"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/hostbox"
+ "github.com/carlosframework/platform/internal/hostclaim"
+ "github.com/carlosframework/platform/internal/moves"
+)
+
+// J1: the claim phases a recovery writes are the ones the wake policy keys
+// on. A recovery probe is admitted only at probing, and a serving wake only
+// at probed or opened, for both recovery kinds; no other phase of either
+// order starts code.
+func TestRecoveryPhasesMeetWakePhase(t *testing.T) {
+ for _, kind := range []string{hostclaim.KindRestoring, hostclaim.KindReverting} {
+ order := hostclaim.RecoveryPhaseOrder(kind)
+ n := len(order)
+ if n < 3 || order[n-3] != hostclaim.PhaseProbing || order[n-2] != hostclaim.PhaseProbed || order[n-1] != hostclaim.PhaseOpened {
+ t.Fatalf("%s order %v does not end probing, probed, opened", kind, order)
+ }
+ phases := order
+ if kind == hostclaim.KindReverting {
+ // A revert inherits the moving phase it began from.
+ phases = append([]string{hostclaim.PhaseMoving, hostclaim.PhaseSealed}, order...)
+ }
+ for _, phase := range phases {
+ c := &hostclaim.Claim{Holder: &hostclaim.Holder{}, Op: hostclaim.Op{Kind: kind, ID: "mv_abcdefghijklmnopqrstuvwxyz", Phase: phase, Exec: "ie-box"}}
+ probe := hostbox.WakePhase(c, "ie-box", true) == nil
+ wake := hostbox.WakePhase(c, "ie-box", false) == nil
+ if probe != (phase == hostclaim.PhaseProbing) {
+ t.Errorf("%s/%q: probe admitted = %v", kind, phase, probe)
+ }
+ if wake != (phase == hostclaim.PhaseProbed || phase == hostclaim.PhaseOpened) {
+ t.Errorf("%s/%q: wake admitted = %v", kind, phase, wake)
+ }
+ }
+ }
+ if hostclaim.RecoveryPhaseOrder(hostclaim.KindLanding) != nil {
+ t.Fatal("landing is not a recovery kind")
+ }
+}
+
+// The rollback id shape is a twin of moves.ValidOpID's rb_ form: a minted
+// rollback id is accepted, a minted move id is not.
+func TestRollBackTakesMintedRollbackIDs(t *testing.T) {
+ landing := func() *hostclaim.Claim {
+ return &hostclaim.Claim{Holder: &hostclaim.Holder{Stamp: 8}, Op: hostclaim.Op{Kind: hostclaim.KindLanding, ID: "mv_abcdefghijklmnopqrstuvwxyz", Phase: "probed", Exec: "uk-box",
+ Intent: map[string]string{"to": "uk", "source": "ie-box", "target": "uk-box"}, Participants: []string{"ie-box", "uk-box"}}}
+ }
+ for i := 0; i < 32; i++ {
+ rb, err := moves.NewRollbackID(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := hostclaim.RollBack("mv_abcdefghijklmnopqrstuvwxyz", rb)(landing()); err != nil {
+ t.Fatalf("%s: %v", rb, err)
+ }
+ mv, err := moves.NewMoveID(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := hostclaim.RollBack("mv_abcdefghijklmnopqrstuvwxyz", mv)(landing()); !errors.Is(err, hostclaim.ErrMalformed) {
+ t.Fatalf("%s: %v", mv, err)
+ }
+ }
+}
diff --git a/internal/hostclaim/move_test.go b/internal/hostclaim/move_test.go
index 20768fd83..465dff847 100644
--- a/internal/hostclaim/move_test.go
+++ b/internal/hostclaim/move_test.go
@@ -1302,3 +1302,770 @@ func TestCommitRefusesAnythingButAProbingTargetLanding(t *testing.T) {
}
}
}
+
+// ---- Plan 4 Task 2: the rollback and the return (§7.8, K4, K8) ----
+
+const (
+ rbA = "rb_abcdefghijklmnopqrstuvwxyz"
+ rbB = "rb_bcdefghijklmnopqrstuvwxyza"
+ stoppedTarget = "moves/bab/app/h/rb/stopped-target.json"
+)
+
+// targetWroteLanding is probedLanding with the target's probe admitted as
+// the writer: holder.generation and writer_box are the target's, and the
+// last admission record is the target's writer (its predecessor the
+// source's generation in ie). The flip at gen 7 made the target's
+// installation stamp F = 8.
+func targetWroteLanding(t *testing.T) *Claim {
+ t.Helper()
+ c := probedLanding(t)
+ c.Holder.Generation, c.Holder.WriterBox = moveA, "uk-box"
+ if c.Holder.Stamp != 8 {
+ t.Fatalf("fixture: the flip's stamp is %d, want 8", c.Holder.Stamp)
+ }
+ return c
+}
+
+// rolledBack is targetWroteLanding after the rollback's claim half.
+func rolledBack(t *testing.T) *Claim {
+ t.Helper()
+ c := targetWroteLanding(t)
+ if err := RollBack(moveA, rbA)(c); err != nil {
+ t.Fatal(err)
+ }
+ return c
+}
+
+func releaseScope(c *Claim) ParkScope {
+ return ParkScope{Location: c.Holder.Location, Incarnation: c.Holder.Incarnation, EvidenceAt: time.Date(2026, 10, 6, 11, 0, 0, 0, time.UTC), RequestLimit: 2 * time.Minute}
+}
+
+// released is rolledBack after the target's release, with no park-local
+// (the target listed no paths of F).
+func released(t *testing.T) *Claim {
+ t.Helper()
+ c := rolledBack(t)
+ if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, releaseScope(c), counter())(c); err != nil {
+ t.Fatal(err)
+ }
+ return c
+}
+
+func validAfterWrite(t *testing.T, c *Claim) {
+ t.Helper()
+ d := *c
+ d.Gen++
+ if err := d.Validate(); err != nil {
+ t.Fatalf("the claim after the write must validate: %v", err)
+ }
+}
+
+// The commit and the rollback race on the claim as on the record: whichever
+// writes first, the other is refused and leaves the claim as it found it.
+func TestRollBackRacesTheCommitOnTheClaim(t *testing.T) {
+ newID := counter()
+ // The commit first.
+ c := targetWroteLanding(t)
+ src, stg := commitScopes(c)
+ if err := Commit(moveA, "ie", "ie-box", "uk", src, stg, newID)(c); err != nil {
+ t.Fatal(err)
+ }
+ before := mustJSON(t, c)
+ if err := RollBack(moveA, rbA)(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) {
+ t.Fatalf("rollback after the commit: %v", err)
+ }
+ // The rollback first.
+ d := targetWroteLanding(t)
+ if err := RollBack(moveA, rbA)(d); err != nil {
+ t.Fatal(err)
+ }
+ if d.Op.Kind != KindRestoring || d.Op.Phase != PhaseReleasing || d.Op.Exec != "uk-box" || d.Op.Intent["rollback"] != rbA ||
+ d.Op.ID != moveA || len(d.Op.Participants) != 2 {
+ t.Fatalf("op after the rollback: %+v", d.Op)
+ }
+ // J15: the target installation's stamp F is journalled for its release.
+ if d.Op.Journal[JournalTargetStamp] != "8" {
+ t.Fatalf("target stamp journal %q", d.Op.Journal[JournalTargetStamp])
+ }
+ if d.Holder.Stamp != 8 || d.Holder.Generation != moveA {
+ t.Fatalf("the rollback touched the holder: %+v", d.Holder)
+ }
+ validAfterWrite(t, d)
+ before = mustJSON(t, d)
+ if err := Commit(moveA, "ie", "ie-box", "uk", src, stg, newID)(d); !errors.Is(err, ErrWrongState) || string(mustJSON(t, d)) != string(before) {
+ t.Fatalf("commit after the rollback: %v", err)
+ }
+ // A replay of the same rollback is a no-op; another rollback id is refused.
+ if err := RollBack(moveA, rbA)(d); err != nil || string(mustJSON(t, d)) != string(before) {
+ t.Fatalf("replay: %v", err)
+ }
+ if err := RollBack(moveA, rbB)(d); !errors.Is(err, ErrWrongState) || string(mustJSON(t, d)) != string(before) {
+ t.Fatalf("another rollback id: %v", err)
+ }
+ // Every landing phase before the commit rolls back.
+ for _, phase := range []string{"checking", "parking", "building", "published", "local", "configured", "restored", "probing", "probed"} {
+ e := targetWroteLanding(t)
+ e.Op.Phase = phase
+ if err := RollBack(moveA, rbA)(e); err != nil || e.Op.Kind != KindRestoring || e.Op.Phase != PhaseReleasing {
+ t.Fatalf("from landing/%s: %v %+v", phase, err, e.Op)
+ }
+ }
+ // Refusals leave the claim alone.
+ for name, tc := range map[string]struct {
+ mutate func(*Claim)
+ op, rb string
+ want error
+ }{
+ "another op": {func(*Claim) {}, moveB, rbA, ErrStaleOp},
+ "a move id": {func(*Claim) {}, moveA, moveB, ErrMalformed},
+ "no id": {func(*Claim) {}, moveA, "", ErrMalformed},
+ "short id": {func(*Claim) {}, moveA, "rb_abc", ErrMalformed},
+ "serving": {func(c *Claim) { c.Op = Op{Kind: KindServing, Exec: "uk-box"} }, moveA, rbA, ErrStaleOp},
+ "moving, sealed": {func(c *Claim) { c.Op.Kind, c.Op.Phase, c.Op.Exec = KindMoving, PhaseSealed, "" }, moveA, rbA, ErrWrongState},
+ "reverting": {func(c *Claim) { c.Op.Kind, c.Op.Phase, c.Op.Exec = KindReverting, "", "ie-box" }, moveA, rbA, ErrWrongState},
+ "landing, no holder": {func(c *Claim) { c.Holder = nil }, moveA, rbA, ErrWrongState},
+ "landing, committed": {func(c *Claim) { c.Op.Phase = PhaseCommitted }, moveA, rbA, ErrWrongState},
+ "restoring under rbB": {func(c *Claim) { c.Op.Kind, c.Op.Phase, c.Op.Intent["rollback"] = KindRestoring, PhaseReleasing, rbB }, moveA, rbA, ErrWrongState},
+ } {
+ e := targetWroteLanding(t)
+ tc.mutate(e)
+ before := mustJSON(t, e)
+ if err := RollBack(tc.op, tc.rb)(e); !errors.Is(err, tc.want) || string(mustJSON(t, e)) != string(before) {
+ t.Fatalf("%s: got %v, want %v", name, err, tc.want)
+ }
+ }
+}
+
+// J16: a record already at landing whose claim is still moving/sealed (the
+// flip's claim half not yet written) rolls back through RollBackFlipped,
+// which writes the flip first, in the same write.
+func TestRollBackFlippedWritesTheFlipFirst(t *testing.T) {
+ c := movingToSealed(t)
+ c.Gen = 7
+ before := mustJSON(t, c)
+ if err := RollBack(moveA, rbA)(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) {
+ t.Fatalf("plain rollback over moving/sealed: %v", err)
+ }
+ // The flip half refuses (the source's stamp is wrong): nothing is written.
+ if err := RollBackFlipped(moveA, rbA, "uk-box", 2, "gen-7")(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) {
+ t.Fatalf("a refused flip half: %v\nbefore %s\nafter %s", err, before, mustJSON(t, c))
+ }
+ if err := RollBackFlipped(moveA, "", "uk-box", 3, "gen-7")(c); !errors.Is(err, ErrMalformed) || string(mustJSON(t, c)) != string(before) {
+ t.Fatalf("a malformed rollback id: %v", err)
+ }
+ if err := RollBackFlipped(moveA, rbA, "uk-box", 3, "gen-7")(c); err != nil {
+ t.Fatal(err)
+ }
+ if c.Op.Kind != KindRestoring || c.Op.Phase != PhaseReleasing || c.Op.Exec != "uk-box" || c.Op.Intent["rollback"] != rbA {
+ t.Fatalf("op: %+v", c.Op)
+ }
+ if c.Holder.Stamp != 8 || c.Op.Journal[JournalTargetStamp] != "8" || c.Op.Journal[JournalSourceStamp] != "3" || c.Op.Journal[JournalSrcGen] != "gen-7" {
+ t.Fatalf("stamps: holder %d journal %v", c.Holder.Stamp, c.Op.Journal)
+ }
+ validAfterWrite(t, c)
+ // Over an already flipped (landing) claim it is RollBack, and a replay is a no-op.
+ d := targetWroteLanding(t)
+ if err := RollBackFlipped(moveA, rbA, "uk-box", 3, "gen-7")(d); err != nil || d.Op.Kind != KindRestoring {
+ t.Fatalf("over landing: %v %+v", err, d.Op)
+ }
+ after := mustJSON(t, d)
+ if err := RollBackFlipped(moveA, rbA, "uk-box", 3, "gen-7")(d); err != nil || string(mustJSON(t, d)) != string(after) {
+ t.Fatalf("replay: %v", err)
+ }
+}
+
+// §7.8 released, claim rev 6.14 item 6 (R12-3), J15.
+func TestTargetReleasedCoversTheTargetRegion(t *testing.T) {
+ build := func(inventory map[string]InventoryEntry) *Claim {
+ c := targetWroteLanding(t)
+ // A restarted probe: its record's predecessor is the dead probe's
+ // generation, in the target's region.
+ c.Holder.Admissions = append(c.Holder.Admissions, AdmissionRecord{Region: "uk", PredecessorRegion: "uk", Session: 10, Box: "uk-box", Generation: "gen-r2", Predecessor: moveA, Mode: "cold"})
+ c.Holder.Session, c.Holder.Generation = 10, "gen-r2"
+ c.Inventory = inventory
+ if err := RollBack(moveA, rbA)(c); err != nil {
+ t.Fatal(err)
+ }
+ return c
+ }
+ withF := map[string]InventoryEntry{"uk-box": {Layout: "nested", Paths: map[string]uint64{"/data/uk": 8}}}
+ c := build(withF)
+ ids := 0
+ newID := func() string { ids++; return fmt.Sprintf("t%d", ids) }
+ holder := *c.Holder
+ if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, releaseScope(c), newID)(c); err != nil {
+ t.Fatal(err)
+ }
+ if c.Op.Kind != KindRestoring || c.Op.Phase != PhaseReleased || c.Op.Exec != "" || c.Op.Journal[JournalEvidenceTarget] != stoppedTarget {
+ t.Fatalf("op: %+v", c.Op)
+ }
+ k := obligationKinds(c)
+ if k["park-replica/uk"] != 1 || k["park-local/uk-box"] != 1 || len(c.Obligations) != 2 {
+ t.Fatalf("obligations: %+v", c.Obligations)
+ }
+ for _, o := range c.Obligations {
+ if o.Op != moveA || (o.Kind == "park-replica" && (o.Park == nil || o.Park.Location != c.Holder.Location || o.Owner != "region:uk")) {
+ t.Fatalf("obligation: %+v", o)
+ }
+ }
+ if c.Holder.Admissions[0].Covered || !c.Holder.Admissions[1].Covered {
+ t.Fatalf("coverage: ie-predecessor %v, uk-predecessor %v", c.Holder.Admissions[0].Covered, c.Holder.Admissions[1].Covered)
+ }
+ if c.Holder.Stamp != holder.Stamp || c.Holder.Session != holder.Session || c.Holder.Generation != holder.Generation || c.Holder.WriterBox != holder.WriterBox {
+ t.Fatalf("the release touched the holder: %+v", c.Holder)
+ }
+ validAfterWrite(t, c)
+ // A replay queues nothing and mints no ids, even once a chore is done.
+ used, before := ids, mustJSON(t, c)
+ if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, releaseScope(c), newID)(c); err != nil || ids != used || string(mustJSON(t, c)) != string(before) {
+ t.Fatalf("replay: %v ids %d->%d", err, used, ids)
+ }
+ c.Obligations = c.Obligations[:1]
+ before = mustJSON(t, c)
+ if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, releaseScope(c), newID)(c); err != nil || ids != used || string(mustJSON(t, c)) != string(before) {
+ t.Fatalf("replay after a completed chore: %v ids %d->%d", err, used, ids)
+ }
+ // J15: no paths of F on the target, no park-local.
+ for name, inv := range map[string]map[string]InventoryEntry{
+ "no inventory": nil,
+ "another stamp": {"uk-box": {Layout: "nested", Paths: map[string]uint64{"/data/uk": 3}}},
+ "F on another box": {"ie-box": {Layout: "nested", Paths: map[string]uint64{"/data/ie": 8}}},
+ } {
+ d := build(inv)
+ if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, releaseScope(d), counter())(d); err != nil {
+ t.Fatalf("%s: %v", name, err)
+ }
+ if k := obligationKinds(d); k["park-replica/uk"] != 1 || len(d.Obligations) != 1 {
+ t.Fatalf("%s: obligations %+v", name, d.Obligations)
+ }
+ }
+ // Refusals leave the claim as found.
+ for name, tc := range map[string]struct {
+ mutate func(*Claim)
+ box, region, ev, op string
+ want error
+ }{
+ "another op": {func(*Claim) {}, "uk-box", "uk", stoppedTarget, moveB, ErrStaleOp},
+ "not rolled back": {func(c *Claim) { c.Op.Kind, c.Op.Phase = KindLanding, "probed" }, "uk-box", "uk", stoppedTarget, moveA, ErrWrongState},
+ "the source's box": {func(*Claim) {}, "ie-box", "uk", stoppedTarget, moveA, ErrWrongState},
+ "the source's region": {func(*Claim) {}, "uk-box", "ie", stoppedTarget, moveA, ErrWrongState},
+ "no evidence": {func(*Claim) {}, "uk-box", "uk", "", moveA, ErrMalformed},
+ "exec is not the target": {func(c *Claim) { c.Op.Exec = "ie-box" }, "uk-box", "uk", stoppedTarget, moveA, ErrWrongState},
+ "released, other evidence": {func(c *Claim) {
+ c.Op.Phase, c.Op.Exec, c.Op.Journal[JournalEvidenceTarget] = PhaseReleased, "", "other"
+ }, "uk-box", "uk", stoppedTarget, moveA, ErrWrongState},
+ } {
+ d := build(withF)
+ tc.mutate(d)
+ before := mustJSON(t, d)
+ if err := TargetReleased(tc.op, tc.box, tc.region, tc.ev, releaseScope(d), counter())(d); !errors.Is(err, tc.want) || string(mustJSON(t, d)) != string(before) {
+ t.Fatalf("%s: got %v, want %v", name, err, tc.want)
+ }
+ }
+ // A location park that is not the holder's location refuses the whole
+ // write: exec, phase and coverage stay as found.
+ d := build(withF)
+ before = mustJSON(t, d)
+ bad := releaseScope(d)
+ bad.Location = "instances/" + d.Host
+ if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, bad, counter())(d); !errors.Is(err, ErrMalformed) || string(mustJSON(t, d)) != string(before) {
+ t.Fatalf("bad scope: %v\nbefore %s\nafter %s", err, before, mustJSON(t, d))
+ }
+}
+
+// Review Focus 3 (wCleanReturnIdentity, INV-16, R11-4): the clean return
+// names the source's generation and stamp again and leaves the admissions,
+// whose last record is the target's writer, for the re-admission to name.
+func TestReturnToSourceCleanNamesSrcGenAndKeepsAdmissions(t *testing.T) {
+ c := released(t)
+ admissions := append([]AdmissionRecord(nil), c.Holder.Admissions...)
+ session, writer := c.Holder.Session, c.Holder.WriterBox
+ if err := ReturnToSource(moveA, "ie-box", true, ReturnFence{})(c); err != nil {
+ t.Fatal(err)
+ }
+ if c.Op.Kind != KindRestoring || c.Op.Phase != PhaseClaimed || c.Op.Exec != "ie-box" {
+ t.Fatalf("op: %+v", c.Op)
+ }
+ if c.Holder.Generation != "gen-7" || c.Holder.Stamp != 3 {
+ t.Fatalf("holder generation %q stamp %d, want gen-7 and 3", c.Holder.Generation, c.Holder.Stamp)
+ }
+ if !reflect.DeepEqual(c.Holder.Admissions, admissions) || c.Holder.Session != session || c.Holder.WriterBox != writer {
+ t.Fatalf("the return touched the admissions: %+v", c.Holder)
+ }
+ if last := c.Holder.Admissions[len(c.Holder.Admissions)-1]; last.Box != "uk-box" || last.Generation != moveA {
+ t.Fatalf("the last admission is not the target's writer: %+v", last)
+ }
+ validAfterWrite(t, c)
+ // A replay is a no-op.
+ before := mustJSON(t, c)
+ if err := ReturnToSource(moveA, "ie-box", true, ReturnFence{})(c); err != nil || string(mustJSON(t, c)) != string(before) {
+ t.Fatalf("replay: %v", err)
+ }
+ // Refusals.
+ for name, tc := range map[string]struct {
+ claim func() *Claim
+ op string
+ source string
+ want error
+ }{
+ "another op": {func() *Claim { return released(t) }, moveB, "ie-box", ErrStaleOp},
+ "the target": {func() *Claim { return released(t) }, moveA, "uk-box", ErrWrongState},
+ "no source": {func() *Claim { return released(t) }, moveA, "", ErrMalformed},
+ "before the release": {func() *Claim { return rolledBack(t) }, moveA, "ie-box", ErrWrongState},
+ "landing": {func() *Claim { return targetWroteLanding(t) }, moveA, "ie-box", ErrWrongState},
+ "no evidence": {func() *Claim {
+ c := released(t)
+ delete(c.Op.Journal, JournalEvidenceTarget)
+ return c
+ }, moveA, "ie-box", ErrWrongState},
+ "no source stamp": {func() *Claim {
+ c := released(t)
+ delete(c.Op.Journal, JournalSourceStamp)
+ return c
+ }, moveA, "ie-box", ErrWrongState},
+ "no source gen": {func() *Claim {
+ c := released(t)
+ delete(c.Op.Journal, JournalSrcGen)
+ return c
+ }, moveA, "ie-box", ErrWrongState},
+ "exec still a box": {func() *Claim {
+ c := released(t)
+ c.Op.Exec = "uk-box"
+ return c
+ }, moveA, "ie-box", ErrWrongState},
+ "claimed by the other": {func() *Claim {
+ c := released(t)
+ c.Op.Phase, c.Op.Exec = PhaseClaimed, "uk-box"
+ return c
+ }, moveA, "ie-box", ErrWrongState},
+ "probing": {func() *Claim {
+ c := released(t)
+ c.Op.Phase, c.Op.Exec = PhaseProbing, "ie-box"
+ return c
+ }, moveA, "ie-box", ErrWrongState},
+ } {
+ c := tc.claim()
+ before := mustJSON(t, c)
+ if err := ReturnToSource(tc.op, tc.source, true, ReturnFence{})(c); !errors.Is(err, tc.want) || string(mustJSON(t, c)) != string(before) {
+ t.Fatalf("%s: got %v, want %v", name, err, tc.want)
+ }
+ }
+}
+
+// A dirty return names the source's stamp again and leaves holder.generation
+// at the target's last admission: the source's probe names the return
+// generation at its cold admission (§7.8, §19.1 table).
+func TestReturnToSourceDirtyLeavesGeneration(t *testing.T) {
+ c := released(t)
+ if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(c); err != nil {
+ t.Fatal(err)
+ }
+ if c.Op.Phase != PhaseClaimed || c.Op.Exec != "ie-box" || c.Holder.Stamp != 3 || c.Holder.Generation != moveA {
+ t.Fatalf("op %+v holder %+v", c.Op, c.Holder)
+ }
+ // src_gen may be the origin (""): the clean return then names it.
+ d := released(t)
+ d.Op.Journal[JournalSrcGen] = ""
+ if err := ReturnToSource(moveA, "ie-box", true, ReturnFence{})(d); err != nil || d.Holder.Generation != "" {
+ t.Fatalf("origin generation: %v %q", err, d.Holder.Generation)
+ }
+}
+
+// declaredLost is rolledBack (restoring/releasing, exec the target) after
+// claim Plan 8's declare-lost transition for the target.
+func declaredLost(t *testing.T) *Claim {
+ t.Helper()
+ c := rolledBack(t)
+ in := LostDeclaration{LossID: "loss_1", ExpectedClaimGen: c.Gen, ExpectedClaimETag: `"etag"`,
+ HolderIncarnation: c.Holder.Incarnation, HolderStamp: c.Holder.Stamp, HolderSession: c.Holder.Session,
+ OpID: moveA, OpKind: KindRestoring, OpPhase: PhaseReleasing, Box: "uk-box", Enrollment: "enroll-uk",
+ QuarantineRevision: 1, FencedAt: time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC),
+ Consequences: LossConsequences{RevokeExecution: true, RemoveParticipation: true, RecoveryIntent: []byte(`{"kind":"lost-target"}`)}}
+ if err := declareLostTransition(in, testLossOwner{})(c); err != nil {
+ t.Fatal(err)
+ }
+ if c.Op.Exec != "none" || !reflect.DeepEqual(c.Op.Participants, []string{"ie-box"}) || c.Op.Journal["loss.loss_1"] == "" || c.Op.Journal[JournalEvidenceTarget] != "" || c.Op.Phase != PhaseReleasing {
+ t.Fatalf("fixture: %+v", c.Op)
+ }
+ return c
+}
+
+// §7.10: an operator's declaration stands in for the target's evidence. The
+// claim is as Plan 8's transition leaves it (exec "none", the target out of
+// participants, loss.<id> journalled, no evidence_target), stepped to
+// released by the resolve door (J16) before the source claims.
+func TestReturnToSourceAfterADeclaredLoss(t *testing.T) {
+ c := declaredLost(t)
+ before := mustJSON(t, c)
+ // J16: the resolve door steps the claim to released first.
+ if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) {
+ t.Fatalf("at releasing: %v", err)
+ }
+ c.Op.Phase = PhaseReleased
+ if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(c); err != nil {
+ t.Fatal(err)
+ }
+ if c.Op.Exec != "ie-box" || c.Op.Phase != PhaseClaimed || c.Holder.Stamp != 3 {
+ t.Fatalf("op %+v stamp %d", c.Op, c.Holder.Stamp)
+ }
+ validAfterWrite(t, c)
+ // The same without the loss entry is refused.
+ d := declaredLost(t)
+ d.Op.Phase = PhaseReleased
+ delete(d.Op.Journal, "loss.loss_1")
+ before = mustJSON(t, d)
+ if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(d); !errors.Is(err, ErrWrongState) || string(mustJSON(t, d)) != string(before) {
+ t.Fatalf("without the loss entry: %v", err)
+ }
+ // A loss entry the claim never recorded, or one that left the target a
+ // participant, is not the target's declaration.
+ e := declaredLost(t)
+ e.Op.Phase = PhaseReleased
+ e.LossRefs = nil
+ if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(e); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("an unrecorded loss: %v", err)
+ }
+ f := declaredLost(t)
+ f.Op.Phase = PhaseReleased
+ f.Op.Participants = []string{"ie-box", "uk-box"}
+ if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(f); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("the target still a participant: %v", err)
+ }
+}
+
+// J5: a termination fence for the target, verified by the caller before
+// Update (VerifyReturnFence), authorises the return when the closure finds
+// the same fence.<target> entry.
+func TestReturnToSourceWithAVerifiedFence(t *testing.T) {
+ const ref = `{"enrollment":"0123456789abcdef0123456789abcdef","key":"control/box-fences/0123456789abcdef0123456789abcdef.json","sha256":"00"}`
+ fenced := func() *Claim {
+ c := rolledBack(t)
+ c.Op.Phase, c.Op.Exec = PhaseReleased, "none"
+ c.Op.Journal["fence.uk-box"] = ref
+ return c
+ }
+ good := ReturnFence{op: moveA, box: "uk-box", ref: ref}
+ c := fenced()
+ if err := ReturnToSource(moveA, "ie-box", true, good)(c); err != nil {
+ t.Fatal(err)
+ }
+ if c.Op.Exec != "ie-box" || c.Op.Phase != PhaseClaimed || c.Holder.Generation != "gen-7" {
+ t.Fatalf("op %+v holder %+v", c.Op, c.Holder)
+ }
+ for name, f := range map[string]ReturnFence{
+ "unverified": {},
+ "another op": {op: moveB, box: "uk-box", ref: ref},
+ "the source's fence": {op: moveA, box: "ie-box", ref: ref},
+ "another journal value": {op: moveA, box: "uk-box", ref: ref + " "},
+ } {
+ d := fenced()
+ before := mustJSON(t, d)
+ if err := ReturnToSource(moveA, "ie-box", true, f)(d); !errors.Is(err, ErrWrongState) || string(mustJSON(t, d)) != string(before) {
+ t.Fatalf("%s: %v", name, err)
+ }
+ }
+ // VerifyReturnFence verifies before it vouches: a journal entry that does
+ // not decode, or no store, is ErrFence.
+ op := fenced().Op
+ if _, err := VerifyReturnFence(context.Background(), nil, nil, op, "uk-box"); !errors.Is(err, ErrFence) {
+ t.Fatalf("no store: %v", err)
+ }
+ op.Journal["fence.uk-box"] = "not json"
+ if _, err := VerifyReturnFence(context.Background(), release.NewDirStore(t.TempDir()), nil, op, "uk-box"); !errors.Is(err, ErrFence) {
+ t.Fatalf("bad journal: %v", err)
+ }
+}
+
+// K8 and J6: the release to serving from each terminal move state, once no
+// chore under the op remains. The executor named is the one executing.
+func TestReleaseMoveWaitsForObligations(t *testing.T) {
+ states := map[string]struct {
+ kind, phase, exec string
+ }{
+ "committed landing": {KindLanding, PhaseCommitted, "uk-box"},
+ "opened rollback": {KindRestoring, PhaseOpened, "ie-box"},
+ "opened revert": {KindReverting, PhaseOpened, "ie-box"},
+ "interim abort": {KindMoving, PhaseAborting, "ie-box"},
+ "interim abort before cut": {KindMoving, PhaseMoving, "ie-box"},
+ }
+ for name, st := range states {
+ c := targetWroteLanding(t)
+ c.Op.Kind, c.Op.Phase, c.Op.Exec = st.kind, st.phase, st.exec
+ c.Obligations = []Obligation{{ID: "o1", Op: moveA, Kind: "park-replica", Region: "ie", Retirement: "r", Owner: "region:ie"}}
+ before := mustJSON(t, c)
+ if err := ReleaseMove(moveA, st.exec)(c); !errors.Is(err, ErrPendingChore) || string(mustJSON(t, c)) != string(before) {
+ t.Fatalf("%s with a chore: %v", name, err)
+ }
+ c.Obligations[0].Op = "another-op"
+ if err := ReleaseMove(moveA, st.exec)(c); err != nil {
+ t.Fatalf("%s: %v", name, err)
+ }
+ if c.Op.Kind != KindServing || c.Op.Exec != st.exec || c.Op.ID != "" || c.Op.Phase != "" || len(c.Op.Participants) != 0 || c.Op.Journal["released_move"] != moveA || len(c.Op.Journal) != 1 {
+ t.Fatalf("%s: op after release %+v", name, c.Op)
+ }
+ if err := RequireReleasedMove(moveA)(c); err != nil {
+ t.Fatalf("%s: %v", name, err)
+ }
+ validAfterWrite(t, c)
+ }
+ // The new states name their executor exactly.
+ for name, st := range states {
+ if st.kind == KindMoving {
+ continue
+ }
+ c := targetWroteLanding(t)
+ c.Op.Kind, c.Op.Phase, c.Op.Exec = st.kind, st.phase, st.exec
+ other := "ie-box"
+ if st.exec == "ie-box" {
+ other = "uk-box"
+ }
+ if err := ReleaseMove(moveA, other)(c); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("%s released to %s: %v", name, other, err)
+ }
+ }
+ // Not before those states: a revert's phase is read from op.phase (J1),
+ // not rev_phase.
+ for name, st := range map[string]struct{ kind, phase string }{
+ "landing probed": {KindLanding, "probed"},
+ "restoring probed": {KindRestoring, PhaseProbed},
+ "restoring claimed": {KindRestoring, PhaseClaimed},
+ "reverting probed": {KindReverting, PhaseProbed},
+ "reverting, rev only": {KindReverting, PhaseSealed},
+ "moving sealed": {KindMoving, PhaseSealed},
+ } {
+ c := targetWroteLanding(t)
+ c.Op.Kind, c.Op.Phase, c.Op.Exec = st.kind, st.phase, "ie-box"
+ if st.phase == PhaseSealed && st.kind == KindReverting {
+ setJournal(c, JournalRevPhase, PhaseOpened)
+ }
+ if err := ReleaseMove(moveA, "ie-box")(c); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("%s: %v", name, err)
+ }
+ }
+}
+
+// revertingClaim is a revert as Plan 2's Revert leaves it: from the cut's
+// stopping (phase "", no files baseline) or from sealed (a baseline).
+func revertingClaim(t *testing.T, sealed bool) *Claim {
+ t.Helper()
+ var c *Claim
+ if sealed {
+ c = movingToSealed(t)
+ } else {
+ c = movingTestClaim(t)
+ for _, tr := range []func(*Claim) error{
+ BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", time.Now()),
+ ExecToNone(moveA, "ie-box", stoppedSource, JournalEvidenceSource),
+ } {
+ if err := tr(c); err != nil {
+ t.Fatal(err)
+ }
+ }
+ }
+ lane := ParkScope{Location: "moves/" + c.Host + "/" + moveA, Incarnation: c.Holder.Incarnation, EvidenceAt: time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC), RequestLimit: time.Minute}
+ if err := Revert(moveA, "ie-box", "uk", &lane, counter())(c); err != nil {
+ t.Fatal(err)
+ }
+ return c
+}
+
+// §7.4 and J1, J8: rev_phase runs evidence, verify, probing, probed, opened;
+// verify only with the files baseline the record keeps from stopped, and
+// straight to probing without it. op.phase mirrors probing, probed and
+// opened in the same write.
+func TestRevPhaseOrder(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ sealed bool
+ baseline bool
+ steps [][2]string
+ pre string
+ }{
+ {"from stopped, with a baseline", true, true, [][2]string{{"evidence", "verify"}, {"verify", "probing"}, {"probing", "probed"}, {"probed", "opened"}}, PhaseSealed},
+ {"from stopping, no baseline", false, false, [][2]string{{"evidence", "probing"}, {"probing", "probed"}, {"probed", "opened"}}, PhaseMoving},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ c := revertingClaim(t, tc.sealed)
+ if c.Op.Journal[JournalRevPhase] != "evidence" || c.Op.Phase != tc.pre {
+ t.Fatalf("fixture: %+v", c.Op)
+ }
+ // The other entry is refused.
+ wrong := "probing"
+ if !tc.baseline {
+ wrong = "verify"
+ }
+ before := mustJSON(t, c)
+ if err := SetRevPhase(moveA, "evidence", wrong, tc.baseline)(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) {
+ t.Fatalf("evidence to %s: %v", wrong, err)
+ }
+ for _, step := range tc.steps {
+ if err := SetRevPhase(moveA, step[0], step[1], tc.baseline)(c); err != nil {
+ t.Fatalf("%s to %s: %v", step[0], step[1], err)
+ }
+ if c.Op.Journal[JournalRevPhase] != step[1] {
+ t.Fatalf("rev_phase %q after %v", c.Op.Journal[JournalRevPhase], step)
+ }
+ want := tc.pre
+ if step[1] != "verify" {
+ want = step[1]
+ }
+ if c.Op.Phase != want || c.Op.Kind != KindReverting || c.Op.Exec != "ie-box" {
+ t.Fatalf("op after %v: %+v", step, c.Op)
+ }
+ validAfterWrite(t, c)
+ // A replay finds the phase moved.
+ before := mustJSON(t, c)
+ if err := SetRevPhase(moveA, step[0], step[1], tc.baseline)(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) {
+ t.Fatalf("replay of %v: %v", step, err)
+ }
+ }
+ // opened is the last phase; ReleaseMove takes it from there.
+ if err := SetRevPhase(moveA, "opened", "probing", tc.baseline)(c); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("after opened: %v", err)
+ }
+ })
+ }
+ // Skips, other kinds, other ops and a phase that disagrees with rev_phase.
+ c := revertingClaim(t, true)
+ for name, tc := range map[string]struct {
+ op, from, to string
+ want error
+ }{
+ "skip verify to probed": {moveA, "evidence", "probed", ErrWrongState},
+ "skip to opened": {moveA, "evidence", "opened", ErrWrongState},
+ "from the wrong phase": {moveA, "verify", "probing", ErrWrongState},
+ "backwards": {moveA, "evidence", "evidence", ErrWrongState},
+ "an unknown phase": {moveA, "evidence", "stuck", ErrWrongState},
+ "another op": {moveB, "evidence", "verify", ErrStaleOp},
+ } {
+ before := mustJSON(t, c)
+ if err := SetRevPhase(tc.op, tc.from, tc.to, true)(c); !errors.Is(err, tc.want) || string(mustJSON(t, c)) != string(before) {
+ t.Fatalf("%s: got %v, want %v", name, err, tc.want)
+ }
+ }
+ d := revertingClaim(t, true)
+ d.Op.Phase = PhaseProbed // disagrees with rev_phase evidence
+ if err := SetRevPhase(moveA, "evidence", "verify", true)(d); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("op.phase ahead of rev_phase: %v", err)
+ }
+ e := rolledBack(t)
+ setJournal(e, JournalRevPhase, "evidence")
+ if err := SetRevPhase(moveA, "evidence", "verify", true)(e); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("a restoring op: %v", err)
+ }
+}
+
+// J1: the source's recovery phases after claimed are mirrored onto the claim
+// op in order: probing (before the first recovery attempt), probed, opened.
+func TestRestoringPhaseOrder(t *testing.T) {
+ c := released(t)
+ if err := SetRestoringPhase(moveA, PhaseReleased, PhaseClaimed)(c); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("released to claimed is ReturnToSource's: %v", err)
+ }
+ if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(c); err != nil {
+ t.Fatal(err)
+ }
+ for _, step := range [][2]string{{PhaseClaimed, PhaseProbing}, {PhaseProbing, PhaseProbed}, {PhaseProbed, PhaseOpened}} {
+ before := mustJSON(t, c)
+ if err := SetRestoringPhase(moveA, step[0], "nonsense")(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) {
+ t.Fatalf("from %s to nonsense: %v", step[0], err)
+ }
+ if err := SetRestoringPhase(moveA, step[0], step[1])(c); err != nil {
+ t.Fatalf("%v: %v", step, err)
+ }
+ if c.Op.Phase != step[1] || c.Op.Exec != "ie-box" || c.Op.Kind != KindRestoring {
+ t.Fatalf("op after %v: %+v", step, c.Op)
+ }
+ if err := SetRestoringPhase(moveA, step[0], step[1])(c); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("replay %v: %v", step, err)
+ }
+ }
+ d := released(t)
+ if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(d); err != nil {
+ t.Fatal(err)
+ }
+ if err := SetRestoringPhase(moveA, PhaseClaimed, PhaseProbed)(d); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("skip: %v", err)
+ }
+ if err := SetRestoringPhase(moveB, PhaseClaimed, PhaseProbing)(d); !errors.Is(err, ErrStaleOp) {
+ t.Fatalf("another op: %v", err)
+ }
+ // The source executes from claimed on; a claim whose exec is not the
+ // source does not step.
+ d.Op.Exec = "uk-box"
+ if err := SetRestoringPhase(moveA, PhaseClaimed, PhaseProbing)(d); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("exec not the source: %v", err)
+ }
+}
+
+// §7.8 probed (dirty): the return lane moves/<host>/<rb id> is parked in the
+// source's region, queued once, and the release to serving waits for it.
+func TestQueueReturnLaneParkAtProbed(t *testing.T) {
+ c := released(t)
+ c.Obligations = nil // the target's chores are done
+ if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(c); err != nil {
+ t.Fatal(err)
+ }
+ lane := releaseScope(c)
+ lane.Location = "moves/" + c.Host + "/" + rbA
+ ids := 0
+ newID := func() string { ids++; return fmt.Sprintf("l%d", ids) }
+ for _, phase := range []string{PhaseClaimed, PhaseProbing} {
+ c.Op.Phase = phase
+ if err := QueueReturnLanePark(moveA, "ie", lane, newID)(c); !errors.Is(err, ErrWrongState) || ids != 0 {
+ t.Fatalf("at %s: %v", phase, err)
+ }
+ }
+ c.Op.Phase = PhaseProbed
+ for name, tc := range map[string]struct {
+ region string
+ location string
+ want error
+ }{
+ "the target's region": {"uk", lane.Location, ErrWrongState},
+ "the move's lane": {"ie", "moves/" + c.Host + "/" + moveA, ErrMalformed},
+ "the live location": {"ie", c.Holder.Location, ErrMalformed},
+ "the staging prefix": {"ie", "moves/" + c.Host, ErrMalformed},
+ } {
+ s := lane
+ s.Location = tc.location
+ before := mustJSON(t, c)
+ if err := QueueReturnLanePark(moveA, tc.region, s, newID)(c); !errors.Is(err, tc.want) || ids != 0 || string(mustJSON(t, c)) != string(before) {
+ t.Fatalf("%s: %v", name, err)
+ }
+ }
+ if err := QueueReturnLanePark(moveA, "ie", lane, newID)(c); err != nil {
+ t.Fatal(err)
+ }
+ if k := obligationKinds(c); k["park-replica/ie"] != 1 || len(c.Obligations) != 1 || c.Obligations[0].Park.Location != lane.Location || c.Obligations[0].Owner != "region:ie" {
+ t.Fatalf("obligations: %+v", c.Obligations)
+ }
+ validAfterWrite(t, c)
+ used := ids
+ if err := QueueReturnLanePark(moveA, "ie", lane, newID)(c); err != nil || ids != used || len(c.Obligations) != 1 {
+ t.Fatalf("replay: %v", err)
+ }
+ other := lane
+ other.RequestLimit = time.Hour
+ if err := QueueReturnLanePark(moveA, "ie", other, newID)(c); !errors.Is(err, ErrWrongState) {
+ t.Fatalf("another scope: %v", err)
+ }
+ // The release waits for the lane park.
+ if err := SetRestoringPhase(moveA, PhaseProbed, PhaseOpened)(c); err != nil {
+ t.Fatal(err)
+ }
+ if err := ReleaseMove(moveA, "ie-box")(c); !errors.Is(err, ErrPendingChore) {
+ t.Fatalf("release before the lane park: %v", err)
+ }
+ // Once the park engine completes the chore, a replay at probed does not
+ // queue it again.
+ c.Obligations = nil
+ c.Op.Phase = PhaseProbed
+ if err := QueueReturnLanePark(moveA, "ie", lane, newID)(c); err != nil || len(c.Obligations) != 0 || ids != used {
+ t.Fatalf("replay after completion: %v %+v", err, c.Obligations)
+ }
+ c.Op.Phase = PhaseOpened
+ if err := ReleaseMove(moveA, "ie-box")(c); err != nil {
+ t.Fatal(err)
+ }
+}