amadan

carlos / platform Public

provision: rollback, revert, stuck and retry transitions (§7.8, §7.10)

MoveRollBack, MoveRollbackTargetPhase, MoveRollbackSourcePhase,
MoveMarkStuck, MoveRetry, MoveRollbackTargetLost,
MoveRollbackLossCommitted, MoveRolledBackClose, MoveRevertReopened and
MoveRevertClose, with the additive journal fields StuckAt, LossRef,
LostAt and ReopenedAt and the distinct target phase lost.

Every rollback and stuck reason is a recorded constant taken through a
switch (ruling J9): a marked landing's reason becomes the rollback's,
and MoveRetry clears only the stuck reason. The three refusals a door
reaches pass through phase1Conflict. Pending copy rows MV-N184 to
MV-N190. provision.ValidRollbackID is the rb_ twin of moveIDShape,
held to moves.ValidOpID by a moves test (ruling J20).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 9315357c871d48af6effa35268b6488bb11893f2 parent 81d0cc6
8 files changed, +1090 −4
  • approved_moves_copy_test.go +1 −0
  • docs/superpowers/plans/2026-10-06-moves-phase1-copy.md +11 −1
  • internal/console/apiinstancemove.go +8 −1
  • internal/console/apiinstancemove_test.go +25 −0
  • internal/moves/ids.go +3 −2
  • internal/moves/ids_test.go +26 −0
  • internal/provision/move_test.go +563 −0
  • internal/provision/provision.go +453 −0
diff --git a/approved_moves_copy_test.go b/approved_moves_copy_test.go
index ef0477bd0..f4535eab6 100644
--- a/approved_moves_copy_test.go
+++ b/approved_moves_copy_test.go
@@ -71,6 +71,7 @@ var movesCopyIDs = []string{
"MV-N169", "MV-N170", "MV-N171", "MV-N172", "MV-N173", "MV-N174",
"MV-N175", "MV-N176", "MV-N177", "MV-N178", "MV-N179", "MV-N180", "MV-N181",
"MV-N182", "MV-N183",
+ "MV-N184", "MV-N185", "MV-N186", "MV-N187", "MV-N188", "MV-N189", "MV-N190",
}
// copyRow is one row of the record. The File cell is the file the sentence is
diff --git a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md
index 40607c9e1..5abfc92ce 100644
--- a/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md
+++ b/docs/superpowers/plans/2026-10-06-moves-phase1-copy.md
@@ -240,7 +240,17 @@ Columns: id, file, text, status.
## Plan 4
-(appended by Plan 4 Task 11)
+(each task's rows in the task that writes them, ruling J23; the rest appended by Plan 4 Task 11)
+
+| id | file | text | status |
+|---|---|---|---|
+| MV-N184 | internal/provision/provision.go | the target box was declared lost | pending |
+| MV-N185 | internal/provision/provision.go | the source no longer matches what it recorded when it stopped | pending |
+| MV-N186 | internal/provision/provision.go | the recovery probe ran out of attempts | pending |
+| MV-N187 | internal/provision/provision.go | the source did not answer within 15 minutes of moving back | pending |
+| MV-N188 | internal/provision/provision.go | the move has committed. it cannot roll back | pending |
+| MV-N189 | internal/provision/provision.go | the move is not stuck. there is nothing to retry | pending |
+| MV-N190 | internal/provision/provision.go | the target holds nothing the move still needs. there is nothing to declare lost | pending |
## Plan 5
diff --git a/internal/console/apiinstancemove.go b/internal/console/apiinstancemove.go
index dd020172d..4a0482b7c 100644
--- a/internal/console/apiinstancemove.go
+++ b/internal/console/apiinstancemove.go
@@ -888,7 +888,8 @@ func legacyAbortOf(seen provision.MoveState) func(*provision.Record) error {
}
// phase1Conflict maps a provision transition refusal to its recorded
-// sentence (copy record MV-N12 to MV-N16, MV-N38 to MV-N46): never the
+// sentence (copy record MV-N12 to MV-N16, MV-N38 to MV-N46, and Plan 4's
+// rollback, retry and declare-lost refusals): never the
// "provision: move state conflict:" prefix, never an em dash. A refusal
// this table does not know answers a fixed sentence, so a new provision
// sentence cannot reach an operator unreviewed.
@@ -906,6 +907,12 @@ func phase1Conflict(err error) string {
msg == "already aborting",
msg == "no move in flight",
msg == moveChangedSentence,
+ // provision.RefusedCommitted, RefusedNotStuck and
+ // RefusedNothingToDeclare (TestPhase1ConflictPassesPlan4Refusals
+ // holds the literals to the constants).
+ msg == "the move has committed. it cannot roll back",
+ msg == "the move is not stuck. there is nothing to retry",
+ msg == "the target holds nothing the move still needs. there is nothing to declare lost",
strings.HasPrefix(msg, "the record's move is ") && strings.Contains(msg, " for incarnation "),
strings.HasPrefix(msg, "a move to ") && strings.Contains(msg, " is already "),
strings.HasPrefix(msg, "the instance is already in "),
diff --git a/internal/console/apiinstancemove_test.go b/internal/console/apiinstancemove_test.go
index 5a8126101..0f1217770 100644
--- a/internal/console/apiinstancemove_test.go
+++ b/internal/console/apiinstancemove_test.go
@@ -1416,3 +1416,28 @@ func rewriteClaim(t *testing.T, fx *consoleFixture, host string, edit func(*host
t.Fatal(err)
}
}
+
+// Plan 4 Task 1 (ruling J20): the rollback, retry and declare-lost refusals
+// reach an operator as their recorded sentences, taken from the transitions
+// themselves, not the fixed sentence.
+func TestPhase1ConflictPassesPlan4Refusals(t *testing.T) {
+ g := provision.MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)}
+ now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
+ const rb = "rb_abcdefghijklmnopqrstuvwxyz"
+ committed := &provision.Record{Region: "uk", Move: &provision.MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: provision.MoveCommitted}}
+ notStuck := &provision.Record{Region: "ie", Move: &provision.MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: provision.MoveReverting}}
+ cleaned := &provision.Record{Region: "ie", Move: &provision.MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: provision.MoveRollingBack,
+ Rollback: &provision.MoveRollback{ID: rb, TargetPhase: provision.RbTargetCleaned}}}
+ for want, err := range map[string]error{
+ provision.RefusedCommitted: provision.MoveRollBack(g, rb, provision.RollbackLandingDeadline, now)(committed),
+ provision.RefusedNotStuck: provision.MoveRetry(g)(notStuck),
+ provision.RefusedNothingToDeclare: provision.MoveRollbackTargetLost(g, "control/losses/x.json", now)(cleaned),
+ } {
+ if !errors.Is(err, provision.ErrMoveConflict) {
+ t.Fatalf("%q: %v", want, err)
+ }
+ if got := phase1Conflict(err); got != want {
+ t.Fatalf("phase1Conflict(%v) = %q, want %q", err, got, want)
+ }
+ }
+}
diff --git a/internal/moves/ids.go b/internal/moves/ids.go
index cd6ce678f..d7aa48c03 100644
--- a/internal/moves/ids.go
+++ b/internal/moves/ids.go
@@ -14,8 +14,9 @@ import (
// opIDShape is §3: a role prefix and 26 lowercase base32 characters (16
// random bytes, no padding).
//
-// Twin of provision.moveIDShape: the copy is forced, because moves imports
-// provision, so provision cannot import moves.
+// Twin of provision.moveIDShape and provision.rollbackIDShape: the copy is
+// forced, because moves imports provision, so provision cannot import
+// moves. TestOpIDShapeTwinsProvision holds the three together.
var opIDShape = regexp.MustCompile(`^(mv|rb)_[a-z2-7]{26}$`)
var lowerBase32 = base32.NewEncoding("abcdefghijklmnopqrstuvwxyz234567").WithPadding(base32.NoPadding)
diff --git a/internal/moves/ids_test.go b/internal/moves/ids_test.go
index 5b0159987..3caba9afb 100644
--- a/internal/moves/ids_test.go
+++ b/internal/moves/ids_test.go
@@ -5,6 +5,8 @@ import (
"regexp"
"strings"
"testing"
+
+ "github.com/carlosframework/platform/internal/provision"
)
func TestOpIDsAreShapedBySpec(t *testing.T) {
@@ -39,3 +41,27 @@ func TestOpIDMintFailsOnShortRandomSource(t *testing.T) {
t.Fatalf("empty reader gave %q, %v", id, err)
}
}
+
+// opIDShape and provision's two shapes are twins (moves imports provision,
+// so the copy is forced): every id minted here passes provision's check for
+// its kind and fails the other's, and ValidOpID agrees with the pair.
+func TestOpIDShapeTwinsProvision(t *testing.T) {
+ for i := 0; i < 64; i++ {
+ mv, err := NewMoveID(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ rb, err := NewRollbackID(rand.Reader)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if !provision.ValidMoveID(mv) || provision.ValidRollbackID(mv) || !provision.ValidRollbackID(rb) || provision.ValidMoveID(rb) {
+ t.Fatalf("minted %q and %q disagree with provision", mv, rb)
+ }
+ }
+ for _, id := range []string{"mv_abcdefghijklmnopqrstuvwxyz", "rb_abcdefghijklmnopqrstuvwxyz", "rb_short", "xx_abcdefghijklmnopqrstuvwxyz", "rb_ABCDEFGHIJKLMNOPQRSTUVWXYZ", ""} {
+ if ValidOpID(id) != (provision.ValidMoveID(id) || provision.ValidRollbackID(id)) {
+ t.Fatalf("ValidOpID(%q) disagrees with provision", id)
+ }
+ }
+}
diff --git a/internal/provision/move_test.go b/internal/provision/move_test.go
index 5030ab441..b49d60c4a 100644
--- a/internal/provision/move_test.go
+++ b/internal/provision/move_test.go
@@ -800,3 +800,566 @@ func TestProbeAttemptRecordsTheCountOnAProbingLanding(t *testing.T) {
t.Fatalf("committed: %v", err)
}
}
+
+// ---- Plan 4 Task 1: rollback, revert, stuck and retry (§7.8, §7.10) ----
+
+const rbTestID = "rb_abcdefghijklmnopqrstuvwxyz"
+
+// rollbackTestLanding is a landing record on the target at phase, inside
+// both deadlines, with the cut stamped ten seconds before now.
+func rollbackTestLanding(g MoveGuard, now time.Time, phase string) *Record {
+ cut, dl, ldl := now.Add(-10*time.Second), now.Add(115*time.Second), now.Add(10*time.Minute)
+ return &Record{Region: "uk", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveLanding, CutPhase: CutFlipped,
+ LandingPhase: phase, CutAt: &cut, CutDeadline: &dl, LandingDeadline: &ldl, Landing: &MoveLandingJournal{InstalledDBSHA256: "sha256:db", Attempts: 2}}}
+}
+
+// rollingBackRecord is a record MoveRollBack has just written.
+func rollingBackRecord(t *testing.T, g MoveGuard, now time.Time) *Record {
+ t.Helper()
+ rec := rollbackTestLanding(g, now, LandProbing)
+ if err := MoveRollBack(g, rbTestID, RollbackProbeAttempts, now)(rec); err != nil {
+ t.Fatal(err)
+ }
+ return rec
+}
+
+func TestRollBackRacesTheCommit(t *testing.T) {
+ g := cutTestGuard()
+ now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
+ // The commit first: the rollback is refused and nothing moves.
+ rec := rollbackTestLanding(g, now, LandProbed)
+ if err := MoveCommit(g, now)(rec); err != nil {
+ t.Fatal(err)
+ }
+ err := MoveRollBack(g, rbTestID, RollbackLandingDeadline, now)(rec)
+ if !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveCommitted || rec.Region != "uk" || rec.Move.Rollback != nil {
+ t.Fatalf("rollback after the commit: %v %+v", err, rec.Move)
+ }
+ if !strings.HasSuffix(err.Error(), ": "+RefusedCommitted) {
+ t.Fatalf("the commit's refusal: %v", err)
+ }
+ // The rollback first: the commit is refused and nothing moves.
+ rec = rollbackTestLanding(g, now, LandProbed)
+ if err := MoveRollBack(g, rbTestID, RollbackLandingDeadline, now)(rec); err != nil {
+ t.Fatal(err)
+ }
+ if err := MoveCommit(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveRollingBack || rec.Move.CommittedAt != nil || rec.Region != "ie" {
+ t.Fatalf("commit after the rollback: %v %+v", err, rec.Move)
+ }
+}
+
+func TestRollBackStartsTheJournal(t *testing.T) {
+ g := cutTestGuard()
+ now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
+ for _, phase := range LandingOrder() {
+ rec := rollbackTestLanding(g, now, phase)
+ if err := MoveRollBack(g, rbTestID, RollbackLandingDeadline, now)(rec); err != nil {
+ t.Fatalf("from %s: %v", phase, err)
+ }
+ m, rb := rec.Move, rec.Move.Rollback
+ if rec.Region != "ie" || m.State != MoveRollingBack || m.LandingPhase != phase || m.Landing == nil || m.Landing.InstalledDBSHA256 != "sha256:db" || m.Reason != "" {
+ t.Fatalf("from %s: %+v", phase, m)
+ }
+ if rb == nil || rb.ID != rbTestID || rb.RequestedAt == nil || !rb.RequestedAt.Equal(now) || rb.Reason != RollbackLandingDeadline ||
+ rb.TargetPhase != RbTargetReleasing || rb.SourcePhase != RbSourceWaiting || rb.ReleaseDeadline == nil || !rb.ReleaseDeadline.Equal(now.Add(15*time.Minute)) ||
+ rb.Dirty || rb.StuckAt != "" || rb.LossRef != "" {
+ t.Fatalf("from %s: rollback %+v", phase, rb)
+ }
+ }
+ // A marked landing: the mark is the rollback's reason, whatever the
+ // caller names (the first reason stands), and Move.Reason is left free
+ // for a stuck reason.
+ for _, reason := range []string{"", RollbackDeclaredLost, RollbackLandingDeadline} {
+ rec := rollbackTestLanding(g, now, LandConfigured)
+ if err := MoveMarkRollback(g, RollbackConfigChanged)(rec); err != nil {
+ t.Fatal(err)
+ }
+ if err := MoveRollBack(g, rbTestID, reason, now)(rec); err != nil || rec.Move.Rollback.Reason != RollbackConfigChanged || rec.Move.Reason != "" {
+ t.Fatalf("marked, caller says %q: %v %+v", reason, err, rec.Move.Rollback)
+ }
+ }
+ // A marked landing still refuses free text from the caller.
+ marked := rollbackTestLanding(g, now, LandConfigured)
+ if err := MoveMarkRollback(g, RollbackConfigChanged)(marked); err != nil {
+ t.Fatal(err)
+ }
+ if err := MoveRollBack(g, rbTestID, "declared lost: box-7", now)(marked); err == nil || marked.Move.State != MoveLanding || marked.Move.Rollback != nil || marked.Move.Reason != RollbackConfigChanged {
+ t.Fatalf("marked, free text: %v %+v", err, marked.Move)
+ }
+ // Every recorded rollback reason is taken; the declared loss is one.
+ for _, reason := range []string{RollbackConfigChanged, RollbackLandingDeadline, RollbackClaimWired, RollbackLandingLayout, RollbackInstallMismatch, RollbackFilesOccupied, RollbackProbeAttempts, RollbackCommitDeadline, RollbackDeclaredLost} {
+ rec := rollbackTestLanding(g, now, LandChecking)
+ if err := MoveRollBack(g, rbTestID, reason, now)(rec); err != nil || rec.Move.Rollback.Reason != reason {
+ t.Fatalf("%q: %v", reason, err)
+ }
+ }
+ // Refusals that write nothing: no reason on an unmarked landing, an
+ // unrecorded reason, a malformed rollback id, a zero time.
+ for name, step := range map[string]func(*Record) error{
+ "no reason": MoveRollBack(g, rbTestID, "", now),
+ "free text": MoveRollBack(g, rbTestID, "declared lost: box-7", now),
+ "move id": MoveRollBack(g, g.ID, RollbackLandingDeadline, now),
+ "short id": MoveRollBack(g, "rb_short", RollbackLandingDeadline, now),
+ "zero time": MoveRollBack(g, rbTestID, RollbackLandingDeadline, time.Time{}),
+ "another guard": MoveRollBack(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, rbTestID, RollbackLandingDeadline, now),
+ } {
+ rec := rollbackTestLanding(g, now, LandChecking)
+ if err := step(rec); err == nil || rec.Move.State != MoveLanding || rec.Region != "uk" || rec.Move.Rollback != nil {
+ t.Fatalf("%s: %v %+v", name, err, rec.Move)
+ }
+ }
+ // Only a landing rolls back: a replay finds rolling_back and is a
+ // conflict, which the caller answers by re-reading.
+ for _, state := range []string{MoveReady, MoveCutting, MoveReverting, MoveRollingBack, MoveStuck, MoveCommitted} {
+ rec := rollbackTestLanding(g, now, LandChecking)
+ rec.Move.State = state
+ if err := MoveRollBack(g, rbTestID, RollbackLandingDeadline, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Region != "uk" || rec.Move.Rollback != nil {
+ t.Fatalf("from %s: %v", state, err)
+ }
+ }
+}
+
+func TestRollbackTargetPhasesInOrder(t *testing.T) {
+ g := cutTestGuard()
+ now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
+ step := func(rec *Record, from, to string, stamp func(*MoveRollback)) error {
+ return MoveRollbackTargetPhase(g, from, to, stamp)(rec)
+ }
+ recovered := func(rb *MoveRollback) { at := now; rb.RecoveredAt = &at }
+
+ // Dirty: every phase, in order; cleaning waits for the source's
+ // recovery (Review Focus 4).
+ rec := rollingBackRecord(t, g, now)
+ if err := step(rec, RbTargetReleasing, RbTargetReturning, nil); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("skip released: %v", err)
+ }
+ if err := step(rec, RbTargetReleasing, RbTargetReleased, func(rb *MoveRollback) { rb.Dirty, rb.Evidence = true, "stopped-target.json" }); err != nil || !rec.Move.Rollback.Dirty {
+ t.Fatal(err)
+ }
+ if err := step(rec, RbTargetReleased, RbTargetCleaning, nil); !errors.Is(err, ErrMoveConflict) || rec.Move.Rollback.TargetPhase != RbTargetReleased {
+ t.Fatalf("a dirty release skipped the return: %v", err)
+ }
+ for _, s := range [][2]string{{RbTargetReleased, RbTargetReturning}, {RbTargetReturning, RbTargetReturned}} {
+ if err := step(rec, s[0], s[1], nil); err != nil {
+ t.Fatalf("%s to %s: %v", s[0], s[1], err)
+ }
+ }
+ if err := step(rec, RbTargetReturned, RbTargetCleaning, nil); !errors.Is(err, ErrMoveConflict) || rec.Move.Rollback.TargetPhase != RbTargetReturned {
+ t.Fatalf("cleaning before the source recovered: %v", err)
+ }
+ recovered(rec.Move.Rollback)
+ if err := step(rec, RbTargetReturned, RbTargetCleaning, nil); err != nil {
+ t.Fatal(err)
+ }
+ if err := step(rec, RbTargetCleaning, RbTargetCleaned, func(rb *MoveRollback) { at := now; rb.TargetParkedAt = &at }); err != nil || rec.Move.Rollback.TargetParkedAt == nil {
+ t.Fatal(err)
+ }
+ if err := step(rec, RbTargetCleaned, RbTargetReleasing, nil); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("past cleaned: %v", err)
+ }
+
+ // Clean: released goes straight to cleaning and never returns.
+ rec = rollingBackRecord(t, g, now)
+ if err := step(rec, RbTargetReleasing, RbTargetReleased, nil); err != nil {
+ t.Fatal(err)
+ }
+ if err := step(rec, RbTargetReleased, RbTargetReturning, nil); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("a clean release returned: %v", err)
+ }
+ recovered(rec.Move.Rollback)
+ if err := step(rec, RbTargetReleased, RbTargetCleaning, nil); err != nil || rec.Move.Rollback.TargetPhase != RbTargetCleaning {
+ t.Fatalf("the clean skip: %v", err)
+ }
+
+ // The wrong from, the wrong guard, a stuck record, a landing: refused.
+ rec = rollingBackRecord(t, g, now)
+ if err := step(rec, RbTargetReleased, RbTargetReturning, nil); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("wrong from: %v", err)
+ }
+ if err := MoveRollbackTargetPhase(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, RbTargetReleasing, RbTargetReleased, nil)(rec); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("another guard: %v", err)
+ }
+ rec.Move.State = MoveStuck
+ if err := step(rec, RbTargetReleasing, RbTargetReleased, nil); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("stuck: %v", err)
+ }
+ if err := step(rollbackTestLanding(g, now, LandProbing), RbTargetReleasing, RbTargetReleased, nil); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("landing: %v", err)
+ }
+}
+
+func TestRollbackTargetLostAndTheLossCommit(t *testing.T) {
+ g := cutTestGuard()
+ now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
+ const ref = "control/losses/a.go.example.com/loss_1.json"
+ at := func(phase string, dirty bool) *Record {
+ rec := rollingBackRecord(t, g, now)
+ rec.Move.Rollback.TargetPhase, rec.Move.Rollback.Dirty = phase, dirty
+ return rec
+ }
+ for _, c := range []struct {
+ phase string
+ dirty bool
+ }{{RbTargetReleasing, false}, {RbTargetReturning, true}, {RbTargetReleased, true}} {
+ rec := at(c.phase, c.dirty)
+ if err := MoveRollbackTargetLost(g, ref, now)(rec); err != nil {
+ t.Fatalf("from %s: %v", c.phase, err)
+ }
+ rb := rec.Move.Rollback
+ if rb.TargetPhase != RbTargetLost || rb.LossRef != ref || rb.LostAt == nil || !rb.LostAt.Equal(now) || rb.Dirty != c.dirty || rec.Move.State != MoveRollingBack {
+ t.Fatalf("from %s: %+v", c.phase, rb)
+ }
+ // A replay of the same declaration changes nothing; another is refused.
+ if err := MoveRollbackTargetLost(g, ref, now.Add(time.Minute))(rec); err != nil || !rb.LostAt.Equal(now) {
+ t.Fatalf("replay from %s: %v", c.phase, err)
+ }
+ if err := MoveRollbackTargetLost(g, ref+"x", now)(rec); !errors.Is(err, ErrMoveConflict) || rb.LossRef != ref {
+ t.Fatalf("another declaration from %s: %v", c.phase, err)
+ }
+ // The forward order never leaves lost, and the source never takes it.
+ for _, to := range []string{RbTargetReleased, RbTargetReturning, RbTargetReturned, RbTargetCleaning} {
+ if err := MoveRollbackTargetPhase(g, RbTargetLost, to, nil)(rec); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("lost to %s: %v", to, err)
+ }
+ }
+ if err := MoveRollbackSourcePhase(g, RbSourceWaiting, RbSourceClaimed, nil)(rec); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("the source consumed lost: %v", err)
+ }
+ // The loss commit: lost to released, clean, evidence the loss record.
+ if err := MoveRollbackLossCommitted(g)(rec); err != nil || rb.TargetPhase != RbTargetReleased || rb.Dirty || rb.Evidence != ref || rb.LossRef != ref {
+ t.Fatalf("loss commit from %s: %v %+v", c.phase, err, rb)
+ }
+ if err := MoveRollbackLossCommitted(g)(rec); err != nil || rb.TargetPhase != RbTargetReleased {
+ t.Fatalf("loss commit replay: %v", err)
+ }
+ if err := MoveRollbackSourcePhase(g, RbSourceWaiting, RbSourceClaimed, nil)(rec); err != nil {
+ t.Fatalf("the source after the loss commit: %v", err)
+ }
+ }
+ // Refused: released clean (nothing left to declare), returned, cleaning,
+ // cleaned; and the loss commit from anything but lost.
+ for _, c := range []struct {
+ phase string
+ dirty bool
+ }{{RbTargetReleased, false}, {RbTargetReturned, true}, {RbTargetCleaning, true}, {RbTargetCleaned, false}} {
+ rec := at(c.phase, c.dirty)
+ err := MoveRollbackTargetLost(g, ref, now)(rec)
+ if !errors.Is(err, ErrMoveConflict) || rec.Move.Rollback.TargetPhase != c.phase || rec.Move.Rollback.LossRef != "" {
+ t.Fatalf("lost from %s: %v", c.phase, err)
+ }
+ if !strings.HasSuffix(err.Error(), ": "+RefusedNothingToDeclare) {
+ t.Fatalf("lost from %s: %v", c.phase, err)
+ }
+ }
+ for _, phase := range []string{RbTargetReleasing, RbTargetReleased, RbTargetReturning, RbTargetReturned} {
+ rec := at(phase, true)
+ if err := MoveRollbackLossCommitted(g)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Rollback.TargetPhase != phase || !rec.Move.Rollback.Dirty {
+ t.Fatalf("loss commit from %s: %v", phase, err)
+ }
+ }
+ rec := at(RbTargetReleasing, false)
+ rec.Move.State = MoveStuck
+ if err := MoveRollbackTargetLost(g, ref, now)(rec); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("stuck: %v", err)
+ }
+ if err := MoveRollbackTargetLost(g, "", now)(at(RbTargetReleasing, false)); err == nil {
+ t.Fatal("an empty loss reference")
+ }
+}
+
+func TestRollbackSourcePhasesCleanAndDirty(t *testing.T) {
+ g := cutTestGuard()
+ now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
+ walk := func(rec *Record, order []string) {
+ t.Helper()
+ for i := 0; i+1 < len(order); i++ {
+ if err := MoveRollbackSourcePhase(g, order[i], order[i+1], nil)(rec); err != nil || rec.Move.Rollback.SourcePhase != order[i+1] {
+ t.Fatalf("%s to %s: %v", order[i], order[i+1], err)
+ }
+ }
+ }
+ // waiting holds until the target's evidence: released clean, returned dirty.
+ rec := rollingBackRecord(t, g, now)
+ if err := MoveRollbackSourcePhase(g, RbSourceWaiting, RbSourceClaimed, nil)(rec); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("waiting while the target releases: %v", err)
+ }
+ rec.Move.Rollback.TargetPhase = RbTargetReleased
+ walk(rec, []string{RbSourceWaiting, RbSourceClaimed, RbSourceRestoring})
+ if err := MoveRollbackSourcePhase(g, RbSourceRestoring, RbSourceParking, nil)(rec); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("a clean return installed: %v", err)
+ }
+ if err := MoveRollbackSourcePhase(g, RbSourceRestoring, RbSourceProbing, nil)(rec); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("a clean return skipped verification: %v", err)
+ }
+ walk(rec, []string{RbSourceRestoring, RbSourceVerified, RbSourceProbing, RbSourceProbed})
+ if err := MoveRollbackSourcePhase(g, RbSourceProbed, RbSourceRecovered, func(rb *MoveRollback) { at := now; rb.RecoveredAt = &at })(rec); err != nil || rec.Move.Rollback.RecoveredAt == nil {
+ t.Fatal(err)
+ }
+ walk(rec, []string{RbSourceRecovered, RbSourceOpened})
+ if err := MoveRollbackSourcePhase(g, RbSourceOpened, RbSourceWaiting, nil)(rec); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("past opened: %v", err)
+ }
+
+ // Dirty: a released dirty target is not yet evidence; returned is.
+ rec = rollingBackRecord(t, g, now)
+ rec.Move.Rollback.TargetPhase, rec.Move.Rollback.Dirty = RbTargetReleased, true
+ if err := MoveRollbackSourcePhase(g, RbSourceWaiting, RbSourceClaimed, nil)(rec); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("waiting while the target has not returned: %v", err)
+ }
+ rec.Move.Rollback.TargetPhase = RbTargetReturned
+ walk(rec, []string{RbSourceWaiting, RbSourceClaimed, RbSourceRestoring})
+ if err := MoveRollbackSourcePhase(g, RbSourceRestoring, RbSourceVerified, nil)(rec); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("a dirty return verified instead of installing: %v", err)
+ }
+ walk(rec, []string{RbSourceRestoring, RbSourceParking, RbSourceBuilding, RbSourcePublished, RbSourceLocal, RbSourceConfigured, RbSourceRestored, RbSourceProbing, RbSourceProbed, RbSourceRecovered, RbSourceOpened})
+
+ // Skips, the wrong guard and a stuck record are refused.
+ rec = rollingBackRecord(t, g, now)
+ rec.Move.Rollback.TargetPhase = RbTargetReleased
+ if err := MoveRollbackSourcePhase(g, RbSourceWaiting, RbSourceRestoring, nil)(rec); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("skip claimed: %v", err)
+ }
+ if err := MoveRollbackSourcePhase(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, RbSourceWaiting, RbSourceClaimed, nil)(rec); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("another guard: %v", err)
+ }
+ rec.Move.State = MoveStuck
+ if err := MoveRollbackSourcePhase(g, RbSourceWaiting, RbSourceClaimed, nil)(rec); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("stuck: %v", err)
+ }
+}
+
+func TestStuckAndRetryReturnToThePhase(t *testing.T) {
+ g := cutTestGuard()
+ now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
+ at := func(phase string) *Record {
+ rec := rollingBackRecord(t, g, now)
+ rec.Move.Rollback.TargetPhase, rec.Move.Rollback.SourcePhase = RbTargetReleased, phase
+ return rec
+ }
+ for _, c := range []struct{ phase, reason string }{
+ {RbSourceRestoring, StuckBaselineMismatch},
+ {RbSourceVerified, StuckBaselineMismatch},
+ {RbSourceProbing, StuckRecoveryProbeAttempts},
+ } {
+ rec := at(c.phase)
+ if err := MoveMarkStuck(g, c.reason)(rec); err != nil {
+ t.Fatalf("%s: %v", c.phase, err)
+ }
+ m, rb := rec.Move, rec.Move.Rollback
+ if m.State != MoveStuck || m.Reason != c.reason || rb.StuckAt != c.phase || rb.SourcePhase != c.phase || rb.Reason != RollbackProbeAttempts {
+ t.Fatalf("%s: %+v %+v", c.phase, m, rb)
+ }
+ // The same mark again is a replay; another is refused.
+ if err := MoveMarkStuck(g, c.reason)(rec); err != nil {
+ t.Fatalf("%s replay: %v", c.phase, err)
+ }
+ other := StuckBaselineMismatch
+ if c.reason == other {
+ other = StuckRecoveryProbeAttempts
+ }
+ if err := MoveMarkStuck(g, other)(rec); !errors.Is(err, ErrMoveConflict) || m.Reason != c.reason {
+ t.Fatalf("%s another reason: %v", c.phase, err)
+ }
+ // Nothing advances a stuck source.
+ if err := MoveRollbackSourcePhase(g, c.phase, RbSourceProbed, nil)(rec); !errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("%s advanced while stuck: %v", c.phase, err)
+ }
+ // Retry: back to rolling_back at the phase it stuck in, so a
+ // mismatch verifies again; the stuck reason goes, the rollback's stays.
+ if err := MoveRetry(g)(rec); err != nil {
+ t.Fatalf("%s retry: %v", c.phase, err)
+ }
+ if m.State != MoveRollingBack || m.Reason != "" || rb.SourcePhase != c.phase || rb.StuckAt != "" || rb.Reason != RollbackProbeAttempts {
+ t.Fatalf("%s after retry: %+v %+v", c.phase, m, rb)
+ }
+ if err := MoveRetry(g)(rec); !errors.Is(err, ErrMoveConflict) || !strings.HasSuffix(err.Error(), ": "+RefusedNotStuck) {
+ t.Fatalf("%s retry twice: %v", c.phase, err)
+ }
+ }
+ // Each stuck reason is the phase's own, and only these phases stick.
+ for _, c := range []struct{ phase, reason string }{
+ {RbSourceProbing, StuckBaselineMismatch},
+ {RbSourceVerified, StuckRecoveryProbeAttempts},
+ {RbSourceVerified, StuckRevertProbeBudget},
+ {RbSourceWaiting, StuckBaselineMismatch},
+ {RbSourceClaimed, StuckBaselineMismatch},
+ {RbSourceParking, StuckBaselineMismatch},
+ {RbSourceProbed, StuckRecoveryProbeAttempts},
+ {RbSourceOpened, StuckRecoveryProbeAttempts},
+ } {
+ rec := at(c.phase)
+ if err := MoveMarkStuck(g, c.reason)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveRollingBack || rec.Move.Rollback.StuckAt != "" {
+ t.Fatalf("%s with %q: %v", c.phase, c.reason, err)
+ }
+ }
+ if err := MoveMarkStuck(g, "the probe failed: connection refused")(at(RbSourceProbing)); err == nil || errors.Is(err, ErrMoveConflict) {
+ t.Fatalf("free text: %v", err)
+ }
+
+ // A revert sticks with no rollback journal and retries to reverting.
+ rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveReverting, CutPhase: CutStaged, Reason: RevertCutDeadline}}
+ for _, reason := range []string{StuckRevertProbeBudget, StuckBaselineMismatch} {
+ rec.Move.State, rec.Move.Reason = MoveReverting, RevertCutDeadline
+ if err := MoveMarkStuck(g, reason)(rec); err != nil || rec.Move.State != MoveStuck || rec.Move.Reason != reason || rec.Move.Rollback != nil {
+ t.Fatalf("revert %q: %v %+v", reason, err, rec.Move)
+ }
+ if err := MoveRetry(g)(rec); err != nil || rec.Move.State != MoveReverting || rec.Move.Reason != "" || rec.Move.Rollback != nil || rec.Move.CutPhase != CutStaged {
+ t.Fatalf("revert retry: %v %+v", err, rec.Move)
+ }
+ }
+ if err := MoveMarkStuck(g, StuckRecoveryProbeAttempts)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveReverting {
+ t.Fatalf("a revert with the recovery probe's reason: %v", err)
+ }
+ // Neither a landing nor a committed move sticks or retries.
+ for _, state := range []string{MoveLanding, MoveCommitted, MoveCutting} {
+ rec := rollbackTestLanding(g, now, LandProbing)
+ rec.Move.State = state
+ if err := MoveMarkStuck(g, StuckRecoveryProbeAttempts)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != state {
+ t.Fatalf("stuck from %s: %v", state, err)
+ }
+ if err := MoveRetry(g)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != state {
+ t.Fatalf("retry from %s: %v", state, err)
+ }
+ }
+ // A stuck rollback that records no phase is not retried blind.
+ rec = at(RbSourceProbing)
+ rec.Move.State, rec.Move.Reason = MoveStuck, StuckRecoveryProbeAttempts
+ if err := MoveRetry(g)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveStuck {
+ t.Fatalf("retry with no stuck phase: %v", err)
+ }
+}
+
+func TestRolledBackCloseNeedsAllThreeStamps(t *testing.T) {
+ g := cutTestGuard()
+ now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
+ later := now.Add(3 * time.Minute)
+ opened := now.Add(50 * time.Second)
+ full := func() *Record {
+ rec := rollingBackRecord(t, g, now)
+ rb := rec.Move.Rollback
+ rb.TargetPhase, rb.SourcePhase = RbTargetCleaned, RbSourceOpened
+ rb.RecoveredAt, rb.OpenedAt, rb.TargetParkedAt = &now, &opened, &now
+ return rec
+ }
+ for name, clear := range map[string]func(*MoveRollback){
+ "recovered": func(rb *MoveRollback) { rb.RecoveredAt = nil },
+ "opened": func(rb *MoveRollback) { rb.OpenedAt = nil },
+ "target parked": func(rb *MoveRollback) { rb.TargetParkedAt = nil },
+ } {
+ rec := full()
+ clear(rec.Move.Rollback)
+ if err := MoveRolledBackClose(g, later)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil || rec.LastMove != nil {
+ t.Fatalf("close without %s: %v", name, err)
+ }
+ }
+ rec := full()
+ rec.Move.State = MoveStuck
+ if err := MoveRolledBackClose(g, later)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil {
+ t.Fatalf("close a stuck record: %v", err)
+ }
+ rec = full()
+ rec.Region = "uk"
+ if err := MoveRolledBackClose(g, later)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil {
+ t.Fatalf("close with the region not the source's: %v", err)
+ }
+ rec = full()
+ if err := MoveRolledBackClose(g, later)(rec); err != nil || rec.Move != nil || rec.Region != "ie" {
+ t.Fatalf("close: %v %+v", err, rec)
+ }
+ // OfflineS is the rollback's open minus the cut: 50 s + 10 s.
+ if lm := rec.LastMove; lm == nil || lm.ID != g.ID || lm.Outcome != "rolled_back" || !lm.At.Equal(later) || lm.OfflineS != 60 {
+ t.Fatalf("last move: %+v", rec.LastMove)
+ }
+}
+
+func TestRevertCloseCountsToTheReopen(t *testing.T) {
+ g := cutTestGuard()
+ now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
+ cut := now.Add(-30 * time.Second)
+ reverting := func() *Record {
+ return &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveReverting, CutPhase: CutStopped, CutAt: &cut, Reason: RevertCutDeadline}}
+ }
+ rec := reverting()
+ if err := MoveRevertClose(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil {
+ t.Fatalf("close before the reopen: %v", err)
+ }
+ if err := MoveRevertReopened(g, now)(rec); err != nil || rec.Move.ReopenedAt == nil || !rec.Move.ReopenedAt.Equal(now) {
+ t.Fatalf("reopened: %v", err)
+ }
+ if err := MoveRevertReopened(g, now.Add(time.Minute))(rec); err != nil || !rec.Move.ReopenedAt.Equal(now) {
+ t.Fatalf("reopened replay keeps the first stamp: %v", err)
+ }
+ if err := MoveRevertClose(g, now.Add(time.Minute))(rec); err != nil || rec.Move != nil || rec.Region != "ie" {
+ t.Fatalf("revert close: %v %+v", err, rec)
+ }
+ if lm := rec.LastMove; lm == nil || lm.ID != g.ID || lm.Outcome != "rolled_back" || lm.OfflineS != 30 || !lm.At.Equal(now.Add(time.Minute)) {
+ t.Fatalf("last move: %+v", rec.LastMove)
+ }
+ // Only a revert reopens or closes this way.
+ for _, state := range []string{MoveCutting, MoveLanding, MoveRollingBack, MoveStuck, MoveCommitted} {
+ rec := reverting()
+ rec.Move.State, rec.Move.ReopenedAt = state, &now
+ if err := MoveRevertClose(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil {
+ t.Fatalf("revert close from %s: %v", state, err)
+ }
+ rec.Move.ReopenedAt = nil
+ if err := MoveRevertReopened(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.ReopenedAt != nil {
+ t.Fatalf("reopened from %s: %v", state, err)
+ }
+ }
+ rec = reverting()
+ rec.Move.ReopenedAt, rec.Region = &now, "uk"
+ if err := MoveRevertClose(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil {
+ t.Fatalf("revert close away from the source: %v", err)
+ }
+}
+
+// The new journal fields survive the record's own write path, under their
+// own keys, and stay off a record that does not use them.
+func TestRollbackJournalThroughTheRecordWrite(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ rec := moveTestRecord(t, s)
+ g := cutTestGuard()
+ now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
+ put := rollbackTestLanding(g, now, LandProbing)
+ ctx := context.Background()
+ if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, func(r *Record) error { r.Region, r.Move = put.Region, put.Move; return nil }); err != nil {
+ t.Fatal(err)
+ }
+ const ref = "control/losses/a.go.example.com/loss_1.json"
+ for _, step := range []func(*Record) error{
+ MoveRollBack(g, rbTestID, RollbackDeclaredLost, now),
+ MoveRollbackTargetLost(g, ref, now),
+ } {
+ if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, step); err != nil {
+ t.Fatal(err)
+ }
+ }
+ got, err := GetRecord(ctx, s, rec.Account, rec.App, rec.Host)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got.Region != "ie" || got.Move.State != MoveRollingBack || got.Move.Rollback.ID != rbTestID || got.Move.Rollback.Reason != RollbackDeclaredLost || got.Move.Rollback.TargetPhase != RbTargetLost || got.Move.Rollback.LossRef != ref || got.Move.Rollback.LostAt == nil {
+ t.Fatalf("read back: %+v %+v", got.Move, got.Move.Rollback)
+ }
+ raw, err := json.Marshal(MoveState{ReopenedAt: &now, Rollback: &MoveRollback{StuckAt: RbSourceProbing, LossRef: ref, LostAt: &now}})
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, key := range []string{`"reopened_at"`, `"stuck_at":"probing"`, `"loss_ref"`, `"lost_at"`} {
+ if !strings.Contains(string(raw), key) {
+ t.Fatalf("%s missing from %s", key, raw)
+ }
+ }
+ if raw, _ := json.Marshal(MoveState{Rollback: &MoveRollback{}}); strings.Contains(string(raw), "stuck_at") || strings.Contains(string(raw), "reopened_at") || strings.Contains(string(raw), "loss_ref") {
+ t.Fatalf("the new keys are not omitempty: %s", raw)
+ }
+}
+
+func TestRollbackIDShape(t *testing.T) {
+ for id, want := range map[string]bool{rbTestID: true, "mv_abcdefghijklmnopqrstuvwxyz": false, "rb_short": false, "rb_ABCDEFGHIJKLMNOPQRSTUVWXYZ": false, "": false} {
+ if ValidRollbackID(id) != want {
+ t.Fatalf("ValidRollbackID(%q) = %v", id, !want)
+ }
+ }
+}
diff --git a/internal/provision/provision.go b/internal/provision/provision.go
index 71345a85a..b8645765c 100644
--- a/internal/provision/provision.go
+++ b/internal/provision/provision.go
@@ -968,6 +968,11 @@ type MoveState struct {
OpenedAt *time.Time `json:"opened_at,omitempty"`
SourceParkedAt *time.Time `json:"source_parked_at,omitempty"`
+ // ReopenedAt is when a revert's source answered its probe again
+ // (rev_phase probed, ruling J26): a revert has no Rollback journal, and
+ // MoveRevertClose counts the offline time from CutAt to this stamp.
+ ReopenedAt *time.Time `json:"reopened_at,omitempty"`
+
Rollback *MoveRollback `json:"rollback,omitempty"`
Reason string `json:"reason,omitempty"`
@@ -1029,6 +1034,15 @@ type MoveRollback struct {
OpenedAt *time.Time `json:"opened_at,omitempty"`
TargetParkedAt *time.Time `json:"target_parked_at,omitempty"`
ReleaseDeadline *time.Time `json:"release_deadline,omitempty"`
+ // StuckAt is the source phase a stuck rollback stopped in (§7.8 stuck);
+ // MoveRetry returns the source to it, so a verification mismatch is
+ // verified again and never bypassed (INV-4). Empty unless stuck.
+ StuckAt string `json:"stuck_at,omitempty"`
+ // LossRef is the declared loss's record key, or the termination fence
+ // record key (§7.10, spec rev 7.19), and LostAt when the record took
+ // it. Written once, by MoveRollbackTargetLost.
+ LossRef string `json:"loss_ref,omitempty"`
+ LostAt *time.Time `json:"lost_at,omitempty"`
// RecoveryInputs are the inputs a recovery install rendered itself
// (§7.7, J2-R2-5); never compared with Inputs.
RecoveryInputs *MoveInputs `json:"recovery_inputs,omitempty"`
@@ -1095,6 +1109,10 @@ const (
RbTargetReturned = "returned"
RbTargetCleaning = "cleaning"
RbTargetCleaned = "cleaned"
+ // RbTargetLost is the declared-loss decision (§7.10), a phase of its
+ // own: no forward step leaves it and the source's waiting never takes
+ // it. Only MoveRollbackLossCommitted moves it on, to released.
+ RbTargetLost = "lost"
RbSourceWaiting = "waiting"
RbSourceClaimed = "claimed"
@@ -1200,6 +1218,15 @@ func MoveStartTransition(p MoveStart) func(*Record) error {
// provision cannot import moves. Keep the two in step.
var moveIDShape = regexp.MustCompile(`^mv_[a-z2-7]{26}$`)
+// rollbackIDShape is a rollback id: "rb_" and 26 lowercase base32
+// characters, the other half of moves.opIDShape (moves.NewRollbackID mints
+// it). Twin of moves.ValidOpID's rollback form, for the same reason as
+// moveIDShape; moves' tests hold the two to each other.
+var rollbackIDShape = regexp.MustCompile(`^rb_[a-z2-7]{26}$`)
+
+// ValidRollbackID reports a rollback id (rollbackIDShape).
+func ValidRollbackID(id string) bool { return rollbackIDShape.MatchString(id) }
+
// MoveGuard is what every Phase 1 transition carries (§7.2): the move id
// and the holder incarnation. A write whose guard does not match the
// record's move is a late write for another move or another incarnation
@@ -1742,6 +1769,432 @@ func MoveClose(g MoveGuard, now time.Time) func(*Record) error {
}
}
+// RollbackDeclaredLost is the rollback reason the declare-lost door
+// records (§7.10): the target box was declared lost before the move
+// committed. A recorded sentence, so it names no box; the box is in the
+// loss record LossRef names.
+const RollbackDeclaredLost = "the target box was declared lost"
+
+// The reasons a move may record when it stops for an operator (§7.8 stuck,
+// §7.4 reverting). Each reaches `carlos move status` through Move.Reason,
+// so each is a recorded sentence with a copy row, and each belongs to the
+// phases MoveMarkStuck names for it.
+const (
+ // StuckBaselineMismatch: the source's files or timers do not hash to
+ // what it recorded when it stopped (INV-4). A rollback's clean return
+ // at restoring or verified, or a revert's verify.
+ StuckBaselineMismatch = "the source no longer matches what it recorded when it stopped"
+ // StuckRecoveryProbeAttempts: a rollback's recovery probe used every
+ // attempt it was given without the app answering (§7.7, nine).
+ StuckRecoveryProbeAttempts = "the recovery probe ran out of attempts"
+ // StuckRevertProbeBudget: a revert's probes failed for 15 minutes.
+ StuckRevertProbeBudget = "the source did not answer within 15 minutes of moving back"
+)
+
+// The refusals of this section that reach an operator through a door
+// (phase1Conflict passes each through; copy record, Plan 4).
+const (
+ // RefusedCommitted: MoveRollBack found the record committed. The commit
+ // won the race (§7.9); the target is authoritative.
+ RefusedCommitted = "the move has committed. it cannot roll back"
+ // RefusedNotStuck: MoveRetry on a move that is not stuck.
+ RefusedNotStuck = "the move is not stuck. there is nothing to retry"
+ // RefusedNothingToDeclare: MoveRollbackTargetLost from a target phase
+ // that is past the window a loss can be declared in.
+ RefusedNothingToDeclare = "the target holds nothing the move still needs. there is nothing to declare lost"
+)
+
+// rollbackReleaseBudget bounds the target's release (§7.8): the console
+// pages on ReleaseDeadline.
+const rollbackReleaseBudget = 15 * time.Minute
+
+// MoveRollBack is the rollback decision (§7.8): a landing, at any landing
+// phase, to rolling_back, in one record write that also moves the region
+// back to From. It is the landing's decision against the commit's: a
+// committed record refuses it (RefusedCommitted), and once it is written
+// the commit is refused, because the state is no longer landing.
+//
+// rbID is the rollback id the caller minted (moves.NewRollbackID). The
+// rollback's reason is the landing's recorded mark when there is one (the
+// first reason stands, as in MoveMarkRollback), else reason, which must be
+// a recorded Rollback constant (a reason given beside a mark must be one
+// too, though the mark wins). Move.Reason is cleared: during a rollback
+// it carries only a stuck reason. LandingPhase and Landing are kept (the
+// target's release reads them). A replay finds rolling_back and is an
+// ErrMoveConflict; the caller re-reads.
+func MoveRollBack(g MoveGuard, rbID, reason string, now time.Time) func(*Record) error {
+ return func(rec *Record) error {
+ if err := g.check(rec); err != nil {
+ return err
+ }
+ m := rec.Move
+ if m.State == MoveCommitted {
+ return fmt.Errorf("%w: %s", ErrMoveConflict, RefusedCommitted)
+ }
+ if m.State != MoveLanding {
+ return fmt.Errorf("%w: a rollback is a landing's", ErrMoveConflict)
+ }
+ if !rollbackIDShape.MatchString(rbID) || now.IsZero() {
+ return fmt.Errorf("provision: malformed rollback %q", rbID)
+ }
+ if reason != "" && !recordedRollbackReason(reason) {
+ return fmt.Errorf("provision: rollback reason %q has no recorded sentence", reason)
+ }
+ why := reason
+ if m.Reason != "" {
+ why = m.Reason
+ }
+ rb := &MoveRollback{ID: rbID, TargetPhase: RbTargetReleasing, SourcePhase: RbSourceWaiting}
+ switch why {
+ case RollbackConfigChanged:
+ rb.Reason = RollbackConfigChanged
+ case RollbackLandingDeadline:
+ rb.Reason = RollbackLandingDeadline
+ case RollbackClaimWired:
+ rb.Reason = RollbackClaimWired
+ case RollbackLandingLayout:
+ rb.Reason = RollbackLandingLayout
+ case RollbackInstallMismatch:
+ rb.Reason = RollbackInstallMismatch
+ case RollbackFilesOccupied:
+ rb.Reason = RollbackFilesOccupied
+ case RollbackProbeAttempts:
+ rb.Reason = RollbackProbeAttempts
+ case RollbackCommitDeadline:
+ rb.Reason = RollbackCommitDeadline
+ case RollbackDeclaredLost:
+ rb.Reason = RollbackDeclaredLost
+ default:
+ return fmt.Errorf("provision: rollback reason %q has no recorded sentence", why)
+ }
+ t := now.UTC()
+ d := t.Add(rollbackReleaseBudget)
+ rb.RequestedAt, rb.ReleaseDeadline = &t, &d
+ rec.Region = m.From
+ m.State, m.Rollback = MoveRollingBack, rb
+ m.Reason = ""
+ return nil
+ }
+}
+
+// recordedRollbackReason reports whether r is one of the sentences
+// MoveRollBack records.
+func recordedRollbackReason(r string) bool {
+ switch r {
+ case RollbackConfigChanged, RollbackLandingDeadline, RollbackClaimWired, RollbackLandingLayout, RollbackInstallMismatch,
+ RollbackFilesOccupied, RollbackProbeAttempts, RollbackCommitDeadline, RollbackDeclaredLost:
+ return true
+ }
+ return false
+}
+
+// rollingBack checks the guard and that the record is a rolling_back move
+// with its journal.
+func rollingBack(g MoveGuard, rec *Record) (*MoveRollback, error) {
+ if err := g.check(rec); err != nil {
+ return nil, err
+ }
+ if rec.Move.State != MoveRollingBack || rec.Move.Rollback == nil {
+ return nil, fmt.Errorf("%w: the move is %s, not rolling back", ErrMoveConflict, rec.Move.State)
+ }
+ return rec.Move.Rollback, nil
+}
+
+// rbTargetOrder is the target's rollback order (§7.8 target table). A
+// clean release skips returning and returned; a dirty one may not.
+var rbTargetOrder = []string{RbTargetReleasing, RbTargetReleased, RbTargetReturning, RbTargetReturned, RbTargetCleaning, RbTargetCleaned}
+
+// MoveRollbackTargetPhase advances the target's rollback phase one step
+// under the guard (§7.8). Beyond the order: released goes to returning
+// only when Dirty and to cleaning only when not, and cleaning waits for the
+// source's RecoveredAt (Review Focus 4). It never leaves lost: a declared
+// loss has its own closures (MoveRollbackLossCommitted,
+// MoveLostTargetParked). stamp runs after the checks.
+func MoveRollbackTargetPhase(g MoveGuard, from, to string, stamp func(*MoveRollback)) func(*Record) error {
+ return func(rec *Record) error {
+ rb, err := rollingBack(g, rec)
+ if err != nil {
+ return err
+ }
+ ok := rb.TargetPhase == from
+ switch {
+ case !ok:
+ case from == RbTargetReleased && to == RbTargetCleaning:
+ ok = !rb.Dirty
+ case from == RbTargetReleased && to == RbTargetReturning:
+ ok = rb.Dirty
+ default:
+ ok = nextIn(rbTargetOrder, from, to)
+ }
+ if ok && to == RbTargetCleaning && rb.RecoveredAt == nil {
+ ok = false
+ }
+ if !ok {
+ return fmt.Errorf("%w: the target's rollback is %s (dirty %t), step expects %s to %s", ErrMoveConflict, rb.TargetPhase, rb.Dirty, from, to)
+ }
+ rb.TargetPhase = to
+ if stamp != nil {
+ stamp(rb)
+ }
+ return nil
+ }
+}
+
+// The source's rollback orders (§7.8 source table): the clean return
+// verifies its own copy, the dirty return installs the target's.
+var (
+ rbSourceHead = []string{RbSourceWaiting, RbSourceClaimed, RbSourceRestoring}
+ rbSourceClean = []string{RbSourceRestoring, RbSourceVerified, RbSourceProbing}
+ rbSourceDirty = []string{RbSourceRestoring, RbSourceParking, RbSourceBuilding, RbSourcePublished, RbSourceLocal, RbSourceConfigured, RbSourceRestored, RbSourceProbing}
+ rbSourceTail = []string{RbSourceProbing, RbSourceProbed, RbSourceRecovered, RbSourceOpened}
+)
+
+// MoveRollbackSourcePhase advances the source's rollback phase one step
+// under the guard (§7.8). The branch after restoring is the target's
+// Dirty: clean verifies, dirty installs. waiting ends only on the target's
+// evidence, released clean or returned dirty; a target at lost is never
+// taken (§7.10), only the released a loss commit writes. stamp runs after
+// the checks.
+func MoveRollbackSourcePhase(g MoveGuard, from, to string, stamp func(*MoveRollback)) func(*Record) error {
+ return func(rec *Record) error {
+ rb, err := rollingBack(g, rec)
+ if err != nil {
+ return err
+ }
+ branch := rbSourceClean
+ if rb.Dirty {
+ branch = rbSourceDirty
+ }
+ ok := rb.SourcePhase == from && (nextIn(rbSourceHead, from, to) || nextIn(branch, from, to) || nextIn(rbSourceTail, from, to))
+ if ok && from == RbSourceWaiting {
+ ok = (rb.TargetPhase == RbTargetReleased && !rb.Dirty) || (rb.TargetPhase == RbTargetReturned && rb.Dirty)
+ }
+ if !ok {
+ return fmt.Errorf("%w: the source's rollback is %s (target %s, dirty %t), step expects %s to %s", ErrMoveConflict, rb.SourcePhase, rb.TargetPhase, rb.Dirty, from, to)
+ }
+ rb.SourcePhase = to
+ if stamp != nil {
+ stamp(rb)
+ }
+ return nil
+ }
+}
+
+// MoveMarkStuck stops a recovery for an operator (§7.8 stuck, §7.4
+// reverting): rolling_back or reverting to stuck, with a recorded reason
+// in Move.Reason. Each reason belongs to its phases:
+//
+// - StuckBaselineMismatch: a rollback whose source is at restoring or
+// verified (a clean return's verification, INV-4), or a revert (its
+// verify);
+// - StuckRecoveryProbeAttempts: a rollback whose source is at probing;
+// - StuckRevertProbeBudget: a revert.
+//
+// A rollback records the source phase in StuckAt, which MoveRetry returns
+// to; a revert has no Rollback journal and resumes from the claim's
+// rev_phase (ruling J20). The same reason again on a stuck record is a
+// replay; another is a conflict. Any other text is refused, and nothing is
+// written.
+func MoveMarkStuck(g MoveGuard, reason string) func(*Record) error {
+ return func(rec *Record) error {
+ if err := g.check(rec); err != nil {
+ return err
+ }
+ m := rec.Move
+ var why string
+ switch reason {
+ case StuckBaselineMismatch:
+ why = StuckBaselineMismatch
+ case StuckRecoveryProbeAttempts:
+ why = StuckRecoveryProbeAttempts
+ case StuckRevertProbeBudget:
+ why = StuckRevertProbeBudget
+ default:
+ return fmt.Errorf("provision: stuck reason %q has no recorded sentence", reason)
+ }
+ if m.State == MoveStuck && m.Reason == why {
+ return nil
+ }
+ ok := false
+ switch m.State {
+ case MoveRollingBack:
+ if rb := m.Rollback; rb != nil {
+ switch why {
+ case StuckBaselineMismatch:
+ ok = rb.SourcePhase == RbSourceRestoring || rb.SourcePhase == RbSourceVerified
+ case StuckRecoveryProbeAttempts:
+ ok = rb.SourcePhase == RbSourceProbing
+ }
+ }
+ case MoveReverting:
+ ok = why == StuckBaselineMismatch || why == StuckRevertProbeBudget
+ }
+ if !ok {
+ return fmt.Errorf("%w: the move cannot stop here for that reason", ErrMoveConflict)
+ }
+ if m.Rollback != nil {
+ m.Rollback.StuckAt = m.Rollback.SourcePhase
+ }
+ m.State = MoveStuck
+ m.Reason = why
+ return nil
+ }
+}
+
+// MoveRetry is the operator's --retry (§7.8, Task 7): stuck back to the
+// recovery it stopped. A rollback returns to rolling_back with the source
+// at the phase StuckAt names, so a mismatch verifies again and a probe
+// probes again; a revert (no Rollback) returns to reverting. It clears the
+// stuck reason (Move.Reason) and StuckAt, never the rollback's own reason.
+func MoveRetry(g MoveGuard) func(*Record) error {
+ return func(rec *Record) error {
+ if err := g.check(rec); err != nil {
+ return err
+ }
+ m := rec.Move
+ if m.State != MoveStuck {
+ return fmt.Errorf("%w: %s", ErrMoveConflict, RefusedNotStuck)
+ }
+ if rb := m.Rollback; rb != nil {
+ if rb.StuckAt == "" {
+ return fmt.Errorf("%w: the stuck rollback records no phase to return to", ErrMoveConflict)
+ }
+ m.State, rb.SourcePhase, rb.StuckAt = MoveRollingBack, rb.StuckAt, ""
+ } else {
+ m.State = MoveReverting
+ }
+ m.Reason = ""
+ return nil
+ }
+}
+
+// MoveRollbackTargetLost is the declared-loss decision on the record
+// (§7.10): the target phase goes to lost, recording the loss record key
+// (or the termination fence record key) and when. Legal while the target
+// still holds something the source needs: releasing, returning, or
+// released dirty and not yet returned. Dirty is left as it was. The same
+// lossRef again is a replay; another is a conflict. The source's recovery
+// does not take lost: MoveRollbackLossCommitted moves it on once the claim
+// carries the declaration.
+func MoveRollbackTargetLost(g MoveGuard, lossRef string, now time.Time) func(*Record) error {
+ return func(rec *Record) error {
+ if lossRef == "" || now.IsZero() {
+ return fmt.Errorf("provision: malformed loss declaration")
+ }
+ rb, err := rollingBack(g, rec)
+ if err != nil {
+ return err
+ }
+ switch {
+ case rb.TargetPhase == RbTargetLost && rb.LossRef == lossRef:
+ return nil
+ case rb.TargetPhase == RbTargetLost:
+ return fmt.Errorf("%w: another loss is recorded for this target", ErrMoveConflict)
+ case rb.TargetPhase == RbTargetReleasing, rb.TargetPhase == RbTargetReturning, rb.TargetPhase == RbTargetReleased && rb.Dirty:
+ default:
+ return fmt.Errorf("%w: %s", ErrMoveConflict, RefusedNothingToDeclare)
+ }
+ t := now.UTC()
+ rb.TargetPhase, rb.LossRef, rb.LostAt = RbTargetLost, lossRef, &t
+ return nil
+ }
+}
+
+// MoveRollbackLossCommitted closes a declared loss on the record (§7.10):
+// lost to released, clean, with LossRef as the evidence, so the source's
+// waiting takes it as a clean release. The caller writes it only after the
+// claim carries the declaration (its loss record committed). A replay
+// finds released with that evidence and changes nothing.
+func MoveRollbackLossCommitted(g MoveGuard) func(*Record) error {
+ return func(rec *Record) error {
+ rb, err := rollingBack(g, rec)
+ if err != nil {
+ return err
+ }
+ if rb.TargetPhase == RbTargetReleased && rb.LossRef != "" && rb.Evidence == rb.LossRef && !rb.Dirty {
+ return nil
+ }
+ if rb.TargetPhase != RbTargetLost || rb.LossRef == "" {
+ return fmt.Errorf("%w: the target's rollback is %s, not lost", ErrMoveConflict, rb.TargetPhase)
+ }
+ rb.TargetPhase, rb.Dirty, rb.Evidence = RbTargetReleased, false, rb.LossRef
+ return nil
+ }
+}
+
+// MoveRolledBackClose is a rollback's close (§7.8 rolled_back, index D11):
+// a rolling_back move whose source has recovered and opened and whose
+// target has parked clears Move and leaves the outcome in LastMove, in the
+// same write. The region is already From (MoveRollBack); a record whose
+// region is not is refused. The offline time is the rollback's OpenedAt
+// minus CutAt, zero when CutAt was never stamped.
+func MoveRolledBackClose(g MoveGuard, now time.Time) func(*Record) error {
+ return func(rec *Record) error {
+ rb, err := rollingBack(g, rec)
+ if err != nil {
+ return err
+ }
+ m := rec.Move
+ if rb.RecoveredAt == nil || rb.OpenedAt == nil || rb.TargetParkedAt == nil {
+ return fmt.Errorf("%w: close follows the recovery, the open and the target park", ErrMoveConflict)
+ }
+ if rec.Region != m.From {
+ return fmt.Errorf("%w: the record's region is not the source's", ErrMoveConflict)
+ }
+ out := &MoveOutcome{ID: g.ID, Outcome: "rolled_back", At: now.UTC()}
+ if m.CutAt != nil {
+ out.OfflineS = rb.OpenedAt.Sub(*m.CutAt).Seconds()
+ }
+ rec.LastMove, rec.Move = out, nil
+ return nil
+ }
+}
+
+// MoveRevertReopened stamps ReopenedAt on a reverting move: its source
+// answered the probe (rev_phase probed, ruling J26). The first stamp
+// stands.
+func MoveRevertReopened(g MoveGuard, now time.Time) func(*Record) error {
+ return func(rec *Record) error {
+ if err := g.check(rec); err != nil {
+ return err
+ }
+ if rec.Move.State != MoveReverting {
+ return fmt.Errorf("%w: the reopen is a revert's", ErrMoveConflict)
+ }
+ if rec.Move.ReopenedAt == nil {
+ t := now.UTC()
+ rec.Move.ReopenedAt = &t
+ }
+ return nil
+ }
+}
+
+// MoveRevertClose is a revert's close (§7.4 reverting, index D11): the
+// caller has seen the claim's rev_phase at opened; the record needs its
+// ReopenedAt. Move is cleared and LastMove says rolled_back, the offline
+// time counted from CutAt to the reopen. A revert never flipped, so the
+// region is From; a record whose region is not is refused.
+func MoveRevertClose(g MoveGuard, now time.Time) func(*Record) error {
+ return func(rec *Record) error {
+ if err := g.check(rec); err != nil {
+ return err
+ }
+ m := rec.Move
+ if m.State != MoveReverting || m.ReopenedAt == nil {
+ return fmt.Errorf("%w: a revert closes after its source reopens", ErrMoveConflict)
+ }
+ if rec.Region != m.From {
+ return fmt.Errorf("%w: the record's region is not the source's", ErrMoveConflict)
+ }
+ out := &MoveOutcome{ID: g.ID, Outcome: "rolled_back", At: now.UTC()}
+ if m.CutAt != nil {
+ out.OfflineS = m.ReopenedAt.Sub(*m.CutAt).Seconds()
+ }
+ rec.LastMove, rec.Move = out, nil
+ return nil
+ }
+}
+
// InstanceStoreEnv is the environment name an instance's private object
// store hangs off: the record's own Environment, or its Slug when it has
// none. Twin of the create door's inline rule (internal/console/