carlos / platform Public

Download this file

1	package provision
2
3	import (
4	"context"
5	"encoding/json"
6	"errors"
7	"strings"
8	"testing"
9	"time"
10
11	"github.com/carlosframework/platform/internal/release"
12	)
13
14	func moveTestRecord(t *testing.T, s release.Store) Record {
15	t.Helper()
16	rec := Record{Schema: 1, Host: "a.go.example.com", Slug: "s", App: "app", Account: "bab", Region: "ie", Channel: "stable", CreatedAt: time.Now().UTC()}
17	if err := CreateRecord(context.Background(), s, rec); err != nil {
18	t.Fatal(err)
19	}
20	return rec
21	}
22
23	func TestMoveStartWritesTheJournalHead(t *testing.T) {
24	s := release.NewDirStore(t.TempDir())
25	rec := moveTestRecord(t, s)
26	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
27	p := MoveStart{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32), From: "ie", To: "uk", Cut: "auto", By: "ops@example.com", Now: now}
28	got, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, MoveStartTransition(p))
29	if err != nil {
30	t.Fatal(err)
31	}
32	m := got.Move
33	if m.ID != p.ID || m.Incarnation != p.Incarnation || m.From != "ie" || m.To != "uk" || m.Cut != "auto" || m.RequestedBy != p.By || m.State != MovePreparing || !m.RequestedAt.Equal(now) || m.UpdatedAt.IsZero() || m.Legacy() {
34	t.Fatalf("journal head: %+v", m)
35	}
36	if got.Region != "ie" {
37	t.Fatal("the request moved the region")
38	}
39	// Refusals: in flight, same region, a replica, a bad id, a bad cut mode.
40	for name, bad := range map[string]MoveStart{
41	"in flight": p,
42	} {
43	if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, MoveStartTransition(bad)); !errors.Is(err, ErrMoveConflict) {
44	t.Fatalf("%s: err %v, want ErrMoveConflict", name, err)
45	}
46	}
47	s2 := release.NewDirStore(t.TempDir())
48	rec2 := moveTestRecord(t, s2)
49	for name, bad := range map[string]MoveStart{
50	"same region": {ID: p.ID, Incarnation: p.Incarnation, From: "ie", To: "ie", Cut: "auto", Now: now},
51	"bad id": {ID: "mv_short", Incarnation: p.Incarnation, From: "ie", To: "uk", Cut: "auto", Now: now},
52	"bad cut": {ID: p.ID, Incarnation: p.Incarnation, From: "ie", To: "uk", Cut: "sometime", Now: now},
53	"wrong from": {ID: p.ID, Incarnation: p.Incarnation, From: "us", To: "uk", Cut: "auto", Now: now},
54	"no inc": {ID: p.ID, To: "uk", From: "ie", Cut: "auto", Now: now},
55	} {
56	if _, err := SetRecordMove(context.Background(), s2, rec2.Account, rec2.App, rec2.Host, MoveStartTransition(bad)); !errors.Is(err, ErrMoveConflict) {
57	t.Fatalf("%s: err %v, want ErrMoveConflict", name, err)
58	}
59	}
60	replica := Record{Schema: 1, Host: "r.go.example.com", Slug: "s", App: "app", Account: "bab", Region: "ie", Channel: "stable", CreatedAt: time.Now().UTC(), Role: "replica"}
61	if err := CreateRecord(context.Background(), s2, replica); err != nil {
62	t.Fatal(err)
63	}
64	if _, err := SetRecordMove(context.Background(), s2, "bab", "app", replica.Host, MoveStartTransition(p)); !errors.Is(err, ErrMoveConflict) {
65	t.Fatalf("replica: %v", err)
66	}
67	}
68
69	// Review Focus 1: a move a v0.22 console wrote has no id. The journal
70	// calls it legacy; the legacy transitions still drive it (the legacy path
71	// stays while the switch is off, Task 0), and no guarded transition
72	// accepts it.
73	func TestMoveStateLegacyRecordIsRefusedByTheGuard(t *testing.T) {
74	s := release.NewDirStore(t.TempDir())
75	rec := moveTestRecord(t, s)
76	if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, MoveRequest("uk", time.Now())); err != nil {
77	t.Fatal(err)
78	}
79	got, _ := GetRecord(context.Background(), s, rec.Account, rec.App, rec.Host)
80	if !got.Move.Legacy() {
81	t.Fatal("an id-less move is legacy")
82	}
83	g := MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)}
84	for name, tr := range map[string]func(*Record) error{
85	"promote": MoveGuardedPromoteReady(g, time.Now(), MoveInputs{}),
86	"cut": MoveGuardedCut(g),
87	"abort": MoveGuardedAbort(g),
88	"abort complete": MoveGuardedAbortComplete(g, time.Now()),
89	// A guard with no id at all must not match an id-less move either.
90	"promote, empty guard": MoveGuardedPromoteReady(MoveGuard{}, time.Now(), MoveInputs{}),
91	} {
92	if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, tr); !errors.Is(err, ErrMoveConflict) {
93	t.Fatalf("guarded %s on a legacy move: %v", name, err)
94	}
95	}
96	// The legacy transitions still work, unchanged.
97	if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, MovePromoteReady(time.Now())); err != nil {
98	t.Fatal(err)
99	}
100	if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, MoveAbort()); err != nil {
101	t.Fatal(err)
102	}
103	if _, err := SetRecordMove(context.Background(), s, rec.Account, rec.App, rec.Host, MoveAbortComplete()); err != nil {
104	t.Fatal(err)
105	}
106	}
107
108	// The switch-off contract: a record written by the legacy transitions
109	// carries exactly the journal keys main wrote, none of the new ones.
110	func TestLegacyMoveRecordKeepsMainsKeys(t *testing.T) {
111	rec := &Record{Region: "ie"}
112	if err := MoveRequest("uk", time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC))(rec); err != nil {
113	t.Fatal(err)
114	}
115	body, err := json.Marshal(rec.Move)
116	if err != nil {
117	t.Fatal(err)
118	}
119	want := `{"to":"uk","state":"preparing","requested_at":"2026-10-06T09:00:00Z","updated_at":"0001-01-01T00:00:00Z"}`
120	if string(body) != want {
121	t.Fatalf("legacy journal drifted:\n got %s\nwant %s", body, want)
122	}
123	out, err := json.Marshal(Record{Region: "ie"})
124	if err != nil {
125	t.Fatal(err)
126	}
127	if strings.Contains(string(out), "last_move") {
128	t.Fatalf("a record at rest grew last_move: %s", out)
129	}
130	}
131
132	func TestGuardedPromoteReadyPinsTheInputs(t *testing.T) {
133	g := MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)}
134	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
135	in := MoveInputs{BundleETags: map[string]string{"app": "e1"}, OverlayETag: "o1", Release: "v1", FilesPlain: "f1", EnvReport: "r1"}
136	rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MovePreparing}}
137	if err := MoveGuardedPromoteReady(g, now, in)(rec); err != nil {
138	t.Fatal(err)
139	}
140	m := rec.Move
141	if m.State != MoveReady || m.ReadyAt == nil || !m.ReadyAt.Equal(now) {
142	t.Fatalf("not ready: %+v", m)
143	}
144	if m.Inputs == nil || m.Inputs.OverlayETag != "o1" || m.Inputs.Release != "v1" || m.Inputs.FilesPlain != "f1" || m.Inputs.EnvReport != "r1" || m.Inputs.BundleETags["app"] != "e1" {
145	t.Fatalf("inputs not pinned: %+v", m.Inputs)
146	}
147	in.BundleETags["app"] = "changed"
148	if m.Inputs.BundleETags["app"] != "e1" {
149	t.Fatal("the pinned inputs alias the caller's map")
150	}
151	// A second promote is refused and does not re-pin.
152	if err := MoveGuardedPromoteReady(g, now, MoveInputs{Release: "v2"})(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Inputs.Release != "v1" {
153	t.Fatalf("second promote: %v %+v", err, rec.Move.Inputs)
154	}
155	// A refused promote leaves a preparing move without inputs.
156	other := &Record{Move: &MoveState{ID: "mv_bbbbbbbbbbbbbbbbbbbbbbbbbb", Incarnation: g.Incarnation, State: MovePreparing}}
157	if err := MoveGuardedPromoteReady(g, now, in)(other); !errors.Is(err, ErrMoveConflict) || other.Move.Inputs != nil || other.Move.State != MovePreparing {
158	t.Fatalf("guard mismatch: %v %+v", err, other.Move)
159	}
160	}
161
162	func TestGuardedCutAndAbortComplete(t *testing.T) {
163	g := MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)}
164	rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MovePreparing}}
165	if err := MoveGuardedCut(g)(rec); !errors.Is(err, ErrMoveConflict) {
166	t.Fatalf("cut from preparing: %v", err)
167	}
168	rec.Move.State = MoveReady
169	// G1: a ready move with no pinned inputs cannot be cut.
170	if err := MoveGuardedCut(g)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveReady || rec.Move.CutPhase != "" {
171	t.Fatalf("cut on nil inputs: %v %+v", err, rec.Move)
172	}
173	rec.Move.Inputs = &MoveInputs{Release: "v1"}
174	if err := MoveGuardedCut(g)(rec); err != nil || rec.Move.State != MoveCutting || rec.Move.CutPhase != CutRequested {
175	t.Fatalf("cut from ready: %v %+v", err, rec.Move)
176	}
177	if err := MoveGuardedCut(g)(rec); !errors.Is(err, ErrMoveConflict) {
178	t.Fatalf("second cut: %v", err)
179	}
180	// Incarnation mismatch alone is refused.
181	stale := MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}
182	if err := MoveGuardedAbort(stale)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveCutting {
183	t.Fatalf("stale incarnation: %v", err)
184	}
185	// Abort from cutting/requested, then abort-complete records the outcome.
186	if err := MoveGuardedAbort(g)(rec); err != nil || rec.Move.State != MoveAborting {
187	t.Fatalf("abort: %v %+v", err, rec.Move)
188	}
189	now := time.Date(2026, 10, 6, 9, 5, 0, 0, time.UTC)
190	if err := MoveGuardedAbortComplete(g, now)(rec); err != nil {
191	t.Fatal(err)
192	}
193	if rec.Move != nil || rec.LastMove == nil || rec.LastMove.ID != g.ID || rec.LastMove.Outcome != "aborted" || !rec.LastMove.At.Equal(now) {
194	t.Fatalf("outcome: move %+v last %+v", rec.Move, rec.LastMove)
195	}
196	if err := MoveGuardedAbortComplete(g, now)(rec); !errors.Is(err, ErrMoveConflict) {
197	t.Fatalf("abort-complete with no move: %v", err)
198	}
199	}
200
201	func TestGuardedAbortLegalStates(t *testing.T) {
202	g := MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)}
203	cases := []struct {
204	state, cutPhase string
205	ok bool
206	}{
207	{MovePreparing, "", true}, {MoveReady, "", true},
208	{MoveCutting, CutRequested, true}, {MoveCutting, CutAdmitted, false},
209	{MoveLanding, "", false}, {MoveRollingBack, "", false}, {MoveReverting, "", false},
210	{MoveAborting, "", false}, {"future", "", false},
211	}
212	for _, c := range cases {
213	m := &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: c.state, CutPhase: c.cutPhase}
214	rec := &Record{Move: m}
215	err := MoveGuardedAbort(g)(rec)
216	if (err == nil) != c.ok {
217	t.Fatalf("abort from %s/%s: err %v", c.state, c.cutPhase, err)
218	}
219	}
220	// The legacy MoveAbort keeps its allowlist (P0's rule): cutting refused.
221	rec := &Record{Move: &MoveState{State: MoveCutting}}
222	if err := MoveAbort()(rec); !errors.Is(err, ErrMoveConflict) {
223	t.Fatal("legacy abort from cutting must stay refused")
224	}
225	}
226
227	func TestGuardedTransitionsRefuseAReplacementMove(t *testing.T) {
228	now := time.Now()
229	gA := MoveGuard{ID: "mv_aaaaaaaaaaaaaaaaaaaaaaaaaa", Incarnation: strings.Repeat("ab", 32)}
230	// A's worker observed preparing; the operator aborted A and requested
231	// B, which is preparing too.
232	rec := &Record{Region: "ie", Move: &MoveState{ID: "mv_bbbbbbbbbbbbbbbbbbbbbbbbbb", Incarnation: gA.Incarnation, From: "ie", To: "uk", State: MovePreparing}}
233	if err := MoveGuardedPromoteReady(gA, now, MoveInputs{})(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MovePreparing {
234	t.Fatalf("A's round promoted B: %v %+v", err, rec.Move)
235	}
236	rec.Move.State = MoveAborting
237	if err := MoveGuardedAbortComplete(gA, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil {
238	t.Fatalf("A's abort cleared B: %v", err)
239	}
240	if rec.LastMove != nil {
241	t.Fatalf("A's refused abort left an outcome: %+v", rec.LastMove)
242	}
243	}
244
245	func TestInstanceStoreEnv(t *testing.T) {
246	if got := InstanceStoreEnv(&Record{Slug: "s"}); got != "s" {
247	t.Fatalf("no environment: %q", got)
248	}
249	if got := InstanceStoreEnv(&Record{Slug: "s", Environment: "canary"}); got != "canary" {
250	t.Fatalf("named environment: %q", got)
251	}
252	}
253
254	func TestManifestReadyPathDefault(t *testing.T) {
255	if (Manifest{}).ReadyPathOrDefault() != "/" {
256	t.Fatal("default ready path")
257	}
258	if (Manifest{Moves: &MovesManifest{ReadyPath: "/healthz"}}).ReadyPathOrDefault() != "/healthz" {
259	t.Fatal("declared ready path")
260	}
261	if (Manifest{Moves: &MovesManifest{}}).ReadyPathOrDefault() != "/" {
262	t.Fatal("empty declaration keeps the default")
263	}
264	}
265
266	func cutTestGuard() MoveGuard {
267	return MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)}
268	}
269
270	func TestCutPhasesAdvanceInOrderUnderTheGuard(t *testing.T) {
271	g := cutTestGuard()
272	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
273	rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveReady, Inputs: &MoveInputs{Release: "v1"}}}
274	if err := MoveGuardedCut(g)(rec); err != nil || rec.Move.State != MoveCutting || rec.Move.CutPhase != CutRequested {
275	t.Fatalf("auto cut: %v %+v", err, rec.Move)
276	}
277	deadline := now.Add(115 * time.Second)
278	if err := MoveAdmit(g, now, 120, deadline, 3)(rec); err != nil || rec.Move.CutPhase != CutAdmitted || !rec.Move.CutDeadline.Equal(deadline) || !rec.Move.CutAt.Equal(now) || rec.Move.CutBudgetS != 120 || rec.Move.Inputs.Release != "v1" {
279	t.Fatalf("admit: %v %+v", err, rec.Move)
280	}
281	// Wrong incarnation: refused.
282	if err := MoveCutPhase(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, CutAdmitted, CutStopping, nil)(rec); !errors.Is(err, ErrMoveConflict) {
283	t.Fatalf("wrong guard: %v", err)
284	}
285	// Skipping a phase: refused.
286	if err := MoveCutPhase(g, CutStopping, CutStopped, nil)(rec); !errors.Is(err, ErrMoveConflict) {
287	t.Fatalf("skip: %v", err)
288	}
289	// Source digests are recorded at stopped only.
290	if err := MoveRecordSource(g, MoveSource{FinalTXID: 1})(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Source != nil {
291	t.Fatalf("source before stopped: %v", err)
292	}
293	for _, step := range [][2]string{{CutAdmitted, CutStopping}, {CutStopping, CutStopped}} {
294	if err := MoveCutPhase(g, step[0], step[1], nil)(rec); err != nil {
295	t.Fatal(err)
296	}
297	}
298	if err := MoveRecordSource(g, MoveSource{FinalTXID: 42, TimersSHA256: "sha256:t"})(rec); err != nil || rec.Move.Source.FinalTXID != 42 {
299	t.Fatalf("source: %v", err)
300	}
301	// A replay of the same transition succeeds, and the record keeps its
302	// own copy of the files digest: the caller mutating its value after
303	// the call does not reach the record (P2-T1-1).
304	files := &MoveFilesDigest{Plain: "sha256:p", Sealed: "sha256:s", Bytes: 7, Entries: 2}
305	src := MoveSource{FinalTXID: 42, TimersSHA256: "sha256:t", Files: files}
306	for i := 0; i < 2; i++ {
307	if err := MoveRecordSource(g, src)(rec); err != nil || rec.Move.Source.FinalTXID != 42 {
308	t.Fatalf("source replay %d: %v", i, err)
309	}
310	}
311	if rec.Move.Source.Files == nil || rec.Move.Source.Files == files {
312	t.Fatalf("the record shares the caller's files digest: %p vs %p", rec.Move.Source.Files, files)
313	}
314	want := MoveFilesDigest{Plain: "sha256:p", Sealed: "sha256:s", Bytes: 7, Entries: 2}
315	files.Plain, files.Sealed, files.Bytes, files.Entries = "x", "y", 99, 99
316	src.Files = nil
317	if got := *rec.Move.Source.Files; got != want {
318	t.Fatalf("the record's files digest followed the caller's: %+v, want %+v", got, want)
319	}
320	for _, step := range [][2]string{{CutStopped, CutStaged}, {CutStaged, CutArchived}} {
321	if err := MoveCutPhase(g, step[0], step[1], nil)(rec); err != nil {
322	t.Fatal(err)
323	}
324	}
325	// Abort is refused past requested.
326	if err := MoveGuardedAbort(g)(rec); !errors.Is(err, ErrMoveConflict) {
327	t.Fatal("abort past admitted")
328	}
329	// The cut cannot step to flipped: only MoveFlip writes it.
330	if err := MoveCutPhase(g, CutArchived, CutFlipped, nil)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.CutPhase != CutArchived {
331	t.Fatalf("a phase step to flipped: %v", err)
332	}
333	ld := now.Add(10 * time.Minute)
334	if err := MoveFlip(g, now, ld)(rec); err != nil || rec.Region != "uk" || rec.Move.State != MoveLanding || rec.Move.CutPhase != CutFlipped || rec.Move.LandingPhase != LandChecking || !rec.Move.LandingDeadline.Equal(ld) || !rec.Move.FlippedAt.Equal(now) {
335	t.Fatalf("flip: %v %+v", err, rec)
336	}
337	if err := MoveFlip(g, now, ld)(rec); !errors.Is(err, ErrMoveConflict) {
338	t.Fatal("a second flip")
339	}
340	if err := MoveLandingPhase(g, LandChecking, LandParking, nil)(rec); err != nil || rec.Move.LandingPhase != LandParking {
341	t.Fatalf("landing phase: %v", err)
342	}
343	if err := MoveLandingPhase(g, LandParking, LandPublished, nil)(rec); !errors.Is(err, ErrMoveConflict) {
344	t.Fatal("a landing phase skipped")
345	}
346	stamped := false
347	if err := MoveLandingPhase(g, LandParking, LandBuilding, func(m *MoveState) { stamped = m.LandingPhase == LandBuilding })(rec); err != nil || !stamped {
348	t.Fatalf("landing stamp: %v", err)
349	}
350	if err := MoveLandingPhase(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, LandBuilding, LandPublished, nil)(rec); !errors.Is(err, ErrMoveConflict) {
351	t.Fatal("landing phase under a stale guard")
352	}
353	}
354
355	// Every legal cut and landing step, and only those, under both orders.
356	func TestCutAndLandingStepsAreTheOrderedPairsOnly(t *testing.T) {
357	g := cutTestGuard()
358	cuts := []string{CutRequested, CutAdmitted, CutStopping, CutStopped, CutStaged, CutArchived}
359	all := append(append([]string{}, cuts...), CutFlipped, "")
360	for _, from := range all {
361	for _, to := range all {
362	rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveCutting, CutPhase: from, Inputs: &MoveInputs{}}}
363	err := MoveCutPhase(g, from, to, nil)(rec)
364	want := false
365	for i := 0; i+1 < len(cuts); i++ {
366	if cuts[i] == from && cuts[i+1] == to {
367	want = true
368	}
369	}
370	// archived to flipped is MoveFlip's, not a phase step's.
371	if from == CutArchived && to == CutFlipped {
372	want = false
373	}
374	if (err == nil) != want {
375	t.Fatalf("cut %q to %q: err %v, want ok=%v", from, to, err, want)
376	}
377	}
378	}
379	lands := []string{LandChecking, LandParking, LandBuilding, LandPublished, LandLocal, LandConfigured, LandRestored, LandProbing, LandProbed}
380	for i, from := range lands {
381	for j, to := range lands {
382	rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveLanding, LandingPhase: from}}
383	err := MoveLandingPhase(g, from, to, nil)(rec)
384	if (err == nil) != (j == i+1) {
385	t.Fatalf("landing %q to %q: err %v", from, to, err)
386	}
387	}
388	}
389	// A landing step from a state that is not landing is refused.
390	rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveCutting, LandingPhase: LandChecking}}
391	if err := MoveLandingPhase(g, LandChecking, LandParking, nil)(rec); !errors.Is(err, ErrMoveConflict) {
392	t.Fatal("landing step in cutting")
393	}
394	}
395
396	func TestBackToReadyAndRevert(t *testing.T) {
397	g := cutTestGuard()
398	now := time.Now()
399	rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveCutting, CutPhase: CutAdmitted, CutAt: &now, CutBudgetS: 120, CutDeadline: &now, Inputs: &MoveInputs{Release: "v1"}}}
400	if err := MoveBackToReady(g)(rec); err != nil || rec.Move.State != MoveReady || rec.Move.CutPhase != "" || rec.Move.CutAt != nil || rec.Move.CutBudgetS != 0 || rec.Move.CutDeadline != nil || rec.Move.Inputs != nil {
401	t.Fatalf("back to ready: %v %+v", err, rec.Move)
402	}
403	if err := MoveBackToReady(g)(rec); !errors.Is(err, ErrMoveConflict) {
404	t.Fatal("back to ready from ready")
405	}
406	for _, ph := range []string{CutStopping, CutStopped, CutStaged, CutArchived} {
407	rec.Move.State, rec.Move.CutPhase = MoveCutting, ph
408	if err := MoveBackToReady(g)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveCutting {
409	t.Fatalf("back to ready from %s: %v", ph, err)
410	}
411	if err := MoveRevert(g, RevertStopEvidenceBudget)(rec); err != nil || rec.Move.State != MoveReverting || rec.Move.Reason != "stop evidence budget" {
412	t.Fatalf("revert from %s: %v %+v", ph, err, rec.Move)
413	}
414	}
415	// Not from requested or admitted: nothing ran yet, so back-to-ready is the exit there.
416	for _, ph := range []string{CutRequested, CutAdmitted, CutFlipped, ""} {
417	rec2 := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveCutting, CutPhase: ph}}
418	if err := MoveRevert(g, "x")(rec2); !errors.Is(err, ErrMoveConflict) || rec2.Move.State != MoveCutting {
419	t.Fatalf("revert from %q: %v", ph, err)
420	}
421	}
422	// Only a recorded sentence can be the reason: anything else is refused
423	// and nothing is written.
424	rec4 := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveCutting, CutPhase: CutStopped}}
425	if err := MoveRevert(g, "some error text")(rec4); err == nil || errors.Is(err, ErrMoveConflict) || rec4.Move.State != MoveCutting || rec4.Move.Reason != "" {
426	t.Fatalf("an unrecorded reason: %v %+v", err, rec4.Move)
427	}
428	// Every recorded reason is accepted and written as itself (Task 5's
429	// reverts: the deadline, the deferred final rounds, the archive).
430	for _, reason := range []string{RevertStopEvidenceBudget, RevertCutDeadline, RevertDeferredRounds, RevertArchiveRefused} {
431	rec5 := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveCutting, CutPhase: CutStaged}}
432	if err := MoveRevert(g, reason)(rec5); err != nil || rec5.Move.State != MoveReverting || rec5.Move.Reason != reason {
433	t.Fatalf("revert for %q: %v %+v", reason, err, rec5.Move)
434	}
435	}
436	// Revert is a cut's recovery only.
437	rec3 := &Record{Region: "uk", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveLanding, CutPhase: CutFlipped}}
438	if err := MoveRevert(g, "x")(rec3); !errors.Is(err, ErrMoveConflict) {
439	t.Fatal("revert from landing")
440	}
441	}
442
443	// G1: the whole loop. Inputs pinned at readiness, drift found at admission,
444	// back to ready (Inputs nil), repin, cut, admit.
445	func TestInputsLoopPinDriftBackRepinCutAdmit(t *testing.T) {
446	g := cutTestGuard()
447	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
448	pin := MoveInputs{BundleETags: map[string]string{"app": "e1"}, Release: "v1"}
449	rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MovePreparing}}
450	if err := MoveGuardedPromoteReady(g, now, pin)(rec); err != nil || rec.Move.Inputs == nil {
451	t.Fatalf("pin: %v", err)
452	}
453	// Repin is legal only with Inputs nil.
454	if err := MoveGuardedRepin(g, MoveInputs{Release: "v2"})(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Inputs.Release != "v1" {
455	t.Fatalf("repin over a pin: %v", err)
456	}
457	if err := MoveGuardedCut(g)(rec); err != nil {
458	t.Fatal(err)
459	}
460	// Drift found: back to ready clears the pin.
461	if err := MoveBackToReady(g)(rec); err != nil || rec.Move.Inputs != nil {
462	t.Fatalf("drift, back to ready: %v %+v", err, rec.Move)
463	}
464	// A ready move with no pin cannot be cut.
465	if err := MoveGuardedCut(g)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveReady {
466	t.Fatalf("cut on nil inputs: %v", err)
467	}
468	// Repin: guarded, legal in ready only.
469	if err := MoveGuardedRepin(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, MoveInputs{Release: "v2"})(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Inputs != nil {
470	t.Fatalf("repin under a stale guard: %v", err)
471	}
472	fresh := MoveInputs{BundleETags: map[string]string{"app": "e2"}, Release: "v2"}
473	if err := MoveGuardedRepin(g, fresh)(rec); err != nil || rec.Move.Inputs == nil || rec.Move.Inputs.Release != "v2" || rec.Move.State != MoveReady {
474	t.Fatalf("repin: %v %+v", err, rec.Move)
475	}
476	fresh.BundleETags["app"] = "changed"
477	if rec.Move.Inputs.BundleETags["app"] != "e2" {
478	t.Fatal("the repin aliases the caller's map")
479	}
480	if err := MoveGuardedRepin(g, MoveInputs{Release: "v3"})(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Inputs.Release != "v2" {
481	t.Fatalf("second repin: %v", err)
482	}
483	if err := MoveGuardedCut(g)(rec); err != nil {
484	t.Fatal(err)
485	}
486	if err := MoveGuardedRepin(g, MoveInputs{Release: "v3"})(rec); !errors.Is(err, ErrMoveConflict) {
487	t.Fatalf("repin in cutting: %v", err)
488	}
489	// Admit pins nothing: it stamps the clock fields and keeps the pin.
490	if err := MoveAdmit(g, now, 120, now.Add(time.Minute), 3)(rec); err != nil || rec.Move.CutPhase != CutAdmitted || rec.Move.Inputs.Release != "v2" {
491	t.Fatalf("admit: %v %+v", err, rec.Move)
492	}
493	// Admit on a cut with no pin is refused and writes nothing.
494	bare := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveCutting, CutPhase: CutRequested}}
495	if err := MoveAdmit(g, now, 120, now.Add(time.Minute), 3)(bare); !errors.Is(err, ErrMoveConflict) || bare.Move.CutPhase != CutRequested || bare.Move.CutAt != nil || bare.Move.CutBudgetS != 0 {
496	t.Fatalf("admit with no pin: %v %+v", err, bare.Move)
497	}
498	}
499
500	// G18: the flip refuses any state but cutting/archived, so a record decision
501	// written first (the C7 conversion's pre-flip guard, a revert, an abort)
502	// wins over a flip that was already in flight.
503	func TestFlipRefusesAnyStateButCuttingArchived(t *testing.T) {
504	g := cutTestGuard()
505	now := time.Now()
506	for _, c := range []struct{ state, phase string }{
507	{MoveReverting, CutArchived}, {MoveAborting, CutArchived}, {MoveRollingBack, CutArchived}, {MoveStuck, CutArchived},
508	{MoveReady, ""}, {MoveCutting, CutStaged}, {MoveCutting, CutRequested}, {MoveCutting, CutFlipped}, {MoveLanding, CutFlipped},
509	} {
510	rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: c.state, CutPhase: c.phase}}
511	if err := MoveFlip(g, now, now.Add(time.Minute))(rec); !errors.Is(err, ErrMoveConflict) || rec.Region != "ie" || rec.Move.State != c.state || rec.Move.FlippedAt != nil {
512	t.Fatalf("flip from %s/%s: %v region %s", c.state, c.phase, err, rec.Region)
513	}
514	}
515	// The sequence that matters: a revert is recorded, then the late flip.
516	rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveCutting, CutPhase: CutArchived}}
517	if err := MoveRevert(g, RevertStopEvidenceBudget)(rec); err != nil {
518	t.Fatal(err)
519	}
520	if err := MoveFlip(g, now, now.Add(time.Minute))(rec); !errors.Is(err, ErrMoveConflict) || rec.Region != "ie" || rec.Move.State != MoveReverting {
521	t.Fatalf("late flip after a revert: %v %+v", err, rec)
522	}
523	// And a stale guard cannot flip an archived cut.
524	rec = &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveCutting, CutPhase: CutArchived}}
525	if err := MoveFlip(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, now, now.Add(time.Minute))(rec); !errors.Is(err, ErrMoveConflict) || rec.Region != "ie" {
526	t.Fatalf("flip under a stale guard: %v", err)
527	}
528	}
529
530	// The transitions run through SetRecordMove end to end, and the flip's
531	// region change and state change land in one record write.
532	func TestFlipIsOneRecordWrite(t *testing.T) {
533	s := release.NewDirStore(t.TempDir())
534	rec := moveTestRecord(t, s)
535	g := cutTestGuard()
536	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
537	ctx := context.Background()
538	seq := []func(*Record) error{
539	MoveStartTransition(MoveStart{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", Cut: "auto", By: "ops@example.com", Now: now}),
540	MoveGuardedPromoteReady(g, now, MoveInputs{Release: "v1"}),
541	MoveGuardedCut(g),
542	MoveAdmit(g, now, 120, now.Add(115*time.Second), 3),
543	MoveCutPhase(g, CutAdmitted, CutStopping, nil),
544	MoveCutPhase(g, CutStopping, CutStopped, nil),
545	MoveCutPhase(g, CutStopped, CutStaged, nil),
546	MoveCutPhase(g, CutStaged, CutArchived, nil),
547	}
548	for i, tr := range seq {
549	if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, tr); err != nil {
550	t.Fatalf("step %d: %v", i, err)
551	}
552	}
553	got, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, MoveFlip(g, now, now.Add(10*time.Minute)))
554	if err != nil || got.Region != "uk" || got.Move.State != MoveLanding {
555	t.Fatalf("flip: %v %+v", err, got)
556	}
557	read, _ := GetRecord(ctx, s, rec.Account, rec.App, rec.Host)
558	if read.Region != "uk" || read.Move.State != MoveLanding || read.Move.LandingPhase != LandChecking || read.Move.Inputs == nil || read.Move.Inputs.Release != "v1" {
559	t.Fatalf("persisted: %+v", read.Move)
560	}
561	}
562
563	// A back-to-ready may record why (Task 5 fix round 1, ruling 1): the
564	// refusal of a holder whose replica layout Phase 1 cannot cut. The reason
565	// is a recorded sentence; anything else is refused and nothing is written.
566	// The next admission clears it, so a reason never outlives the refusal.
567	func TestBackToReadyRecordsARefusal(t *testing.T) {
568	g := cutTestGuard()
569	now := time.Now()
570	rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveCutting, CutPhase: CutAdmitted, CutAt: &now, CutBudgetS: 120, CutDeadline: &now, Inputs: &MoveInputs{Release: "v1"}}}
571	if err := MoveBackToReadyRefused(g, "x")(rec); err == nil || rec.Move.State != MoveCutting || rec.Move.Reason != "" {
572	t.Fatalf("an unrecorded reason: %v %+v", err, rec.Move)
573	}
574	if err := MoveBackToReadyRefused(g, RefusedReplicaLayout)(rec); err != nil || rec.Move.State != MoveReady || rec.Move.CutPhase != "" || rec.Move.Inputs != nil || rec.Move.Reason != RefusedReplicaLayout {
575	t.Fatalf("refused: %v %+v", err, rec.Move)
576	}
577	if err := MoveBackToReadyRefused(g, RefusedReplicaLayout)(rec); !errors.Is(err, ErrMoveConflict) {
578	t.Fatal("refused from ready")
579	}
580	rec.Move.Inputs = &MoveInputs{Release: "v1"}
581	rec.Move.State, rec.Move.CutPhase = MoveCutting, CutRequested
582	if err := MoveAdmit(g, now, 120, now, 3)(rec); err != nil || rec.Move.Reason != "" {
583	t.Fatalf("the admission did not clear the reason: %v %+v", err, rec.Move)
584	}
585	}
586
587	// The cut guard (Paul's ruling, 2026-10-11): an unarmed target or an
588	// unarmed source sends the cut back to ready from requested or admitted,
589	// and each records its own sentence, not the replica layout's.
590	func TestBackToReadyRecordsTheCutGuardRefusals(t *testing.T) {
591	g := cutTestGuard()
592	now := time.Now()
593	for _, reason := range []string{RefusedTargetUnarmed, RefusedSourceUnarmed} {
594	for _, phase := range []string{CutRequested, CutAdmitted} {
595	rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveCutting, CutPhase: phase, CutAt: &now, CutBudgetS: 120, CutDeadline: &now, Inputs: &MoveInputs{Release: "v1"}}}
596	if err := MoveBackToReadyRefused(g, reason)(rec); err != nil || rec.Move.State != MoveReady || rec.Move.CutPhase != "" || rec.Move.Inputs != nil || rec.Move.Reason != reason {
597	t.Fatalf("%s at %s: %v %+v", reason, phase, err, rec.Move)
598	}
599	}
600	}
601	if RefusedTargetUnarmed == RefusedSourceUnarmed || RefusedTargetUnarmed == RefusedReplicaLayout || RefusedSourceUnarmed == RefusedReplicaLayout {
602	t.Fatal("the refusals do not each have their own sentence")
603	}
604	}
605
606	// Fix round 1: a repin clears a recorded refusal. A move is pinned again
607	// only once the refusal's cause is gone (decideReady and T13), so from then
608	// on it is ready to cut and `carlos move status` no longer says the cut did
609	// not start.
610	func TestRepinClearsARecordedRefusal(t *testing.T) {
611	g := cutTestGuard()
612	now := time.Now()
613	rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveCutting, CutPhase: CutRequested, CutAt: &now, Inputs: &MoveInputs{Release: "v1"}}}
614	if err := MoveBackToReadyRefused(g, RefusedSourceUnarmed)(rec); err != nil || rec.Move.Reason != RefusedSourceUnarmed {
615	t.Fatalf("fixture: %v %+v", err, rec.Move)
616	}
617	if err := MoveGuardedRepin(g, MoveInputs{Release: "v2"})(rec); err != nil || rec.Move.Inputs == nil || rec.Move.Reason != "" {
618	t.Fatalf("the repin kept the refusal: %v %+v", err, rec.Move)
619	}
620	}
621
622	// The landing marks a rollback it cannot yet start (Plan 2 Task 10): the
623	// reason is written on the landing record, the state and phase stay as they
624	// are, and Plan 4's rollback transition is the one call site that replaces
625	// it. Only a recorded sentence is accepted; a second write of the same
626	// sentence is a no-op, a different one is refused; and nothing but a landing
627	// record takes it.
628	func TestMarkRollbackRecordsASentenceOnALandingRecord(t *testing.T) {
629	g := cutTestGuard()
630	rec := &Record{Region: "uk", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveLanding, LandingPhase: LandConfigured}}
631	if err := MoveMarkRollback(g, "some error text")(rec); err == nil || errors.Is(err, ErrMoveConflict) || rec.Move.Reason != "" {
632	t.Fatalf("an unrecorded reason: %v %+v", err, rec.Move)
633	}
634	for _, reason := range []string{RollbackConfigChanged, RollbackLandingDeadline, RollbackClaimWired, RollbackLandingLayout, RollbackInstallMismatch, RollbackFilesOccupied} {
635	rec := &Record{Region: "uk", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveLanding, LandingPhase: LandChecking}}
636	if err := MoveMarkRollback(g, reason)(rec); err != nil || rec.Move.State != MoveLanding || rec.Move.LandingPhase != LandChecking || rec.Move.Reason != reason {
637	t.Fatalf("%q: %v %+v", reason, err, rec.Move)
638	}
639	if err := MoveMarkRollback(g, reason)(rec); err != nil || rec.Move.Reason != reason {
640	t.Fatalf("%q again: %v %+v", reason, err, rec.Move)
641	}
642	if err := MoveMarkRollback(g, RollbackLandingDeadline)(rec); reason != RollbackLandingDeadline && (!errors.Is(err, ErrMoveConflict) || rec.Move.Reason != reason) {
643	t.Fatalf("%q then another: %v %+v", reason, err, rec.Move)
644	}
645	}
646	for _, state := range []string{MoveCutting, MoveReady, MoveReverting, MoveCommitted} {
647	rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: state}}
648	if err := MoveMarkRollback(g, RollbackLandingDeadline)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Reason != "" {
649	t.Fatalf("from %s: %v %+v", state, err, rec.Move)
650	}
651	}
652	if got := LandingOrder(); len(got) != 9 || got[0] != LandChecking || got[8] != LandProbed {
653	t.Fatalf("landing order: %v", got)
654	}
655	got := LandingOrder()
656	got[0] = "x"
657	if LandingOrder()[0] != LandChecking {
658	t.Fatal("LandingOrder hands out the package's slice")
659	}
660	}
661
662	func commitTestRecord(g MoveGuard, dl time.Time) *Record {
663	return &Record{Region: "uk", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveLanding, LandingPhase: LandProbed, CutDeadline: &dl}}
664	}
665
666	func TestCommitRefusedPastTheDeadline(t *testing.T) {
667	g := MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)}
668	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
669	dl := now.Add(115 * time.Second)
670	cut := now.Add(-10 * time.Second)
671	rec := &Record{Region: "uk", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveLanding, LandingPhase: LandProbed, CutDeadline: &dl, CutAt: &cut}}
672	if err := MoveCommit(g, dl.Add(time.Second))(rec); !errors.Is(err, ErrCutDeadline) || !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveLanding {
673	t.Fatalf("late commit: %v %s", err, rec.Move.State)
674	}
675	if err := MoveCommit(g, dl)(rec); err != nil || rec.Move.State != MoveCommitted || rec.Move.CommittedAt == nil {
676	t.Fatalf("commit at the deadline: %v", err)
677	}
678	if err := MoveCommit(g, dl)(rec); !errors.Is(err, ErrMoveConflict) {
679	t.Fatal("second commit")
680	}
681	if err := MoveClose(g, now)(rec); !errors.Is(err, ErrMoveConflict) {
682	t.Fatal("close before open and park")
683	}
684	if err := MoveOpened(g, now)(rec); err != nil || rec.Move.OpenedAt == nil {
685	t.Fatal(err)
686	}
687	if err := MoveClose(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil {
688	t.Fatalf("close before the source park: %v", err)
689	}
690	if err := MoveSourceParked(g, now)(rec); err != nil {
691	t.Fatal(err)
692	}
693	if err := MoveClose(g, now)(rec); err != nil || rec.Move != nil || rec.Region != "uk" || rec.LastMove == nil || rec.LastMove.ID != g.ID || rec.LastMove.Outcome != "done" {
694	t.Fatalf("close: %v %+v", err, rec)
695	}
696	if rec.LastMove.OfflineS != 10 || !rec.LastMove.At.Equal(now) {
697	t.Fatalf("last move: %+v", rec.LastMove)
698	}
699	}
700
701	// Ruling H8: a recorded rollback wins over a commit, and the earlier of the
702	// cut and landing deadlines bounds it.
703	func TestCommitRefusesARecordedRollbackAndTheEarlierDeadline(t *testing.T) {
704	g := cutTestGuard()
705	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
706	cutDL, landDL := now.Add(115*time.Second), now.Add(60*time.Second)
707	rec := commitTestRecord(g, cutDL)
708	rec.Move.LandingDeadline = &landDL
709	if err := MoveCommit(g, landDL.Add(time.Second))(rec); !errors.Is(err, ErrCutDeadline) || rec.Move.State != MoveLanding {
710	t.Fatalf("past the landing deadline, inside the cut's: %v", err)
711	}
712	if err := MoveCommit(g, landDL)(rec); err != nil || rec.Move.State != MoveCommitted {
713	t.Fatalf("at the earlier deadline: %v", err)
714	}
715	// The landing deadline alone does not stand in for the cut's.
716	rec = commitTestRecord(g, landDL)
717	rec.Move.LandingDeadline = &cutDL
718	if err := MoveCommit(g, landDL.Add(time.Second))(rec); !errors.Is(err, ErrCutDeadline) {
719	t.Fatalf("past the cut deadline, inside the landing's: %v", err)
720	}
721	// A recorded rollback reason refuses the commit; nothing is written.
722	rec = commitTestRecord(g, cutDL)
723	if err := MoveMarkRollback(g, RollbackLandingDeadline)(rec); err != nil {
724	t.Fatal(err)
725	}
726	if err := MoveCommit(g, now)(rec); !errors.Is(err, ErrMoveConflict) || errors.Is(err, ErrCutDeadline) || rec.Move.State != MoveLanding || rec.Move.CommittedAt != nil {
727	t.Fatalf("commit over a rollback marker: %v %+v", err, rec.Move)
728	}
729	// No cut deadline recorded: the commit cannot be shown to be in time.
730	rec = commitTestRecord(g, cutDL)
731	rec.Move.CutDeadline = nil
732	if err := MoveCommit(g, now)(rec); !errors.Is(err, ErrCutDeadline) {
733	t.Fatalf("no cut deadline: %v", err)
734	}
735	// Not probed yet, a replacement move, and a record with no move.
736	for _, phase := range []string{LandChecking, LandRestored, LandProbing} {
737	rec = commitTestRecord(g, cutDL)
738	rec.Move.LandingPhase = phase
739	if err := MoveCommit(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveLanding {
740	t.Fatalf("commit from %s: %v", phase, err)
741	}
742	}
743	other := MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}
744	if err := MoveCommit(other, now)(commitTestRecord(g, cutDL)); !errors.Is(err, ErrMoveConflict) {
745	t.Fatalf("another incarnation: %v", err)
746	}
747	if err := MoveCommit(g, now)(&Record{}); !errors.Is(err, ErrMoveConflict) {
748	t.Fatalf("no move: %v", err)
749	}
750	}
751
752	func TestOpenedParkedAndCloseAreCommittedOnly(t *testing.T) {
753	g := cutTestGuard()
754	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
755	later := now.Add(time.Minute)
756	for _, state := range []string{MoveLanding, MoveCutting, MoveReverting, MoveRollingBack} {
757	for name, step := range map[string]func(*Record) error{"opened": MoveOpened(g, now), "parked": MoveSourceParked(g, now), "close": MoveClose(g, now)} {
758	rec := commitTestRecord(g, now)
759	rec.Move.State = state
760	if err := step(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil || rec.Move.OpenedAt != nil || rec.Move.SourceParkedAt != nil {
761	t.Fatalf("%s from %s: %v", name, state, err)
762	}
763	}
764	}
765	// A replay keeps the first stamp.
766	rec := commitTestRecord(g, now)
767	rec.Move.State = MoveCommitted
768	if err := MoveOpened(g, now)(rec); err != nil {
769	t.Fatal(err)
770	}
771	if err := MoveOpened(g, later)(rec); err != nil || !rec.Move.OpenedAt.Equal(now) {
772	t.Fatalf("opened replay: %v %v", err, rec.Move.OpenedAt)
773	}
774	if err := MoveSourceParked(g, now)(rec); err != nil {
775	t.Fatal(err)
776	}
777	if err := MoveSourceParked(g, later)(rec); err != nil || !rec.Move.SourceParkedAt.Equal(now) {
778	t.Fatalf("parked replay: %v %v", err, rec.Move.SourceParkedAt)
779	}
780	// Close with no cut_at recorded (a record from before it was stamped):
781	// the outcome is written and the offline time is left at zero.
782	if err := MoveClose(g, later)(rec); err != nil || rec.Move != nil || rec.LastMove == nil || rec.LastMove.OfflineS != 0 {
783	t.Fatalf("close without cut_at: %v %+v", err, rec.LastMove)
784	}
785	// Close refuses a record whose region is not the target's.
786	rec = commitTestRecord(g, now)
787	rec.Move.State, rec.Move.OpenedAt, rec.Move.SourceParkedAt = MoveCommitted, &now, &now
788	rec.Region = "ie"
789	if err := MoveClose(g, later)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil || rec.LastMove != nil {
790	t.Fatalf("close with the region still the source's: %v", err)
791	}
792	}
793
794	func TestProbeAttemptRecordsTheCountOnAProbingLanding(t *testing.T) {
795	g := cutTestGuard()
796	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
797	rec := commitTestRecord(g, now)
798	rec.Move.LandingPhase = LandProbing
799	for _, n := range []int{1, 1, 2, 3} {
800	if err := MoveProbeAttempt(g, n)(rec); err != nil || rec.Move.Landing == nil || rec.Move.Landing.Attempts != n {
801	t.Fatalf("attempt %d: %v %+v", n, err, rec.Move.Landing)
802	}
803	}
804	rec.Move.Landing.InstalledDBSHA256 = "keep"
805	if err := MoveProbeAttempt(g, 4)(rec); err != nil || rec.Move.Landing.InstalledDBSHA256 != "keep" {
806	t.Fatalf("the install digest was lost: %v %+v", err, rec.Move.Landing)
807	}
808	for _, n := range []int{3, 0, -1} {
809	if err := MoveProbeAttempt(g, n)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Landing.Attempts != 4 {
810	t.Fatalf("attempt %d after 4: %v %+v", n, err, rec.Move.Landing)
811	}
812	}
813	for _, phase := range []string{LandChecking, LandRestored, LandProbed} {
814	rec := commitTestRecord(g, now)
815	rec.Move.LandingPhase = phase
816	if err := MoveProbeAttempt(g, 1)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Landing != nil {
817	t.Fatalf("from %s: %v", phase, err)
818	}
819	}
820	rec = commitTestRecord(g, now)
821	rec.Move.LandingPhase = LandProbing
822	if err := MoveMarkRollback(g, RollbackProbeAttempts)(rec); err != nil || rec.Move.Reason != RollbackProbeAttempts {
823	t.Fatalf("out of attempts: %v %q", err, rec.Move.Reason)
824	}
825	if err := MoveProbeAttempt(g, 1)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Landing != nil {
826	t.Fatalf("an attempt over a rollback marker: %v", err)
827	}
828	rec = commitTestRecord(g, now)
829	if err := MoveMarkRollback(g, RollbackCommitDeadline)(rec); err != nil || rec.Move.Reason != RollbackCommitDeadline {
830	t.Fatalf("commit past the deadline: %v %q", err, rec.Move.Reason)
831	}
832	rec = commitTestRecord(g, now)
833	rec.Move.State = MoveCommitted
834	if err := MoveProbeAttempt(g, 1)(rec); !errors.Is(err, ErrMoveConflict) {
835	t.Fatalf("committed: %v", err)
836	}
837	}
838
839	// ---- Plan 4 Task 1: rollback, revert, stuck and retry (§7.8, §7.10) ----
840
841	const rbTestID = "rb_abcdefghijklmnopqrstuvwxyz"
842
843	// rollbackTestLanding is a landing record on the target at phase, inside
844	// both deadlines, with the cut stamped ten seconds before now.
845	func rollbackTestLanding(g MoveGuard, now time.Time, phase string) *Record {
846	cut, dl, ldl := now.Add(-10*time.Second), now.Add(115*time.Second), now.Add(10*time.Minute)
847	return &Record{Region: "uk", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveLanding, CutPhase: CutFlipped,
848	LandingPhase: phase, CutAt: &cut, CutDeadline: &dl, LandingDeadline: &ldl, Landing: &MoveLandingJournal{InstalledDBSHA256: "sha256:db", Attempts: 2}}}
849	}
850
851	// rollingBackRecord is a record MoveRollBack has just written.
852	func rollingBackRecord(t *testing.T, g MoveGuard, now time.Time) *Record {
853	t.Helper()
854	rec := rollbackTestLanding(g, now, LandProbing)
855	if err := MoveRollBack(g, rbTestID, RollbackProbeAttempts, now)(rec); err != nil {
856	t.Fatal(err)
857	}
858	return rec
859	}
860
861	func TestRollBackRacesTheCommit(t *testing.T) {
862	g := cutTestGuard()
863	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
864	// The commit first: the rollback is refused and nothing moves.
865	rec := rollbackTestLanding(g, now, LandProbed)
866	if err := MoveCommit(g, now)(rec); err != nil {
867	t.Fatal(err)
868	}
869	err := MoveRollBack(g, rbTestID, RollbackLandingDeadline, now)(rec)
870	if !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveCommitted || rec.Region != "uk" || rec.Move.Rollback != nil {
871	t.Fatalf("rollback after the commit: %v %+v", err, rec.Move)
872	}
873	if !strings.HasSuffix(err.Error(), ": "+RefusedCommitted) {
874	t.Fatalf("the commit's refusal: %v", err)
875	}
876	// The rollback first: the commit is refused and nothing moves.
877	rec = rollbackTestLanding(g, now, LandProbed)
878	if err := MoveRollBack(g, rbTestID, RollbackLandingDeadline, now)(rec); err != nil {
879	t.Fatal(err)
880	}
881	if err := MoveCommit(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveRollingBack || rec.Move.CommittedAt != nil || rec.Region != "ie" {
882	t.Fatalf("commit after the rollback: %v %+v", err, rec.Move)
883	}
884	}
885
886	func TestRollBackStartsTheJournal(t *testing.T) {
887	g := cutTestGuard()
888	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
889	for _, phase := range LandingOrder() {
890	rec := rollbackTestLanding(g, now, phase)
891	if err := MoveRollBack(g, rbTestID, RollbackLandingDeadline, now)(rec); err != nil {
892	t.Fatalf("from %s: %v", phase, err)
893	}
894	m, rb := rec.Move, rec.Move.Rollback
895	if rec.Region != "ie" || m.State != MoveRollingBack || m.LandingPhase != phase || m.Landing == nil || m.Landing.InstalledDBSHA256 != "sha256:db" || m.Reason != "" {
896	t.Fatalf("from %s: %+v", phase, m)
897	}
898	if rb == nil || rb.ID != rbTestID || rb.RequestedAt == nil || !rb.RequestedAt.Equal(now) || rb.Reason != RollbackLandingDeadline ||
899	rb.TargetPhase != RbTargetReleasing || rb.SourcePhase != RbSourceWaiting || rb.ReleaseDeadline == nil || !rb.ReleaseDeadline.Equal(now.Add(15*time.Minute)) ||
900	rb.Dirty || rb.StuckAt != "" || rb.LossRef != "" {
901	t.Fatalf("from %s: rollback %+v", phase, rb)
902	}
903	}
904	// A marked landing: the mark is copied to the rollback's reason, whatever
905	// the caller names (the first reason stands), and stays on the move.
906	for _, reason := range []string{"", RollbackDeclaredLost, RollbackLandingDeadline} {
907	rec := rollbackTestLanding(g, now, LandConfigured)
908	if err := MoveMarkRollback(g, RollbackConfigChanged)(rec); err != nil {
909	t.Fatal(err)
910	}
911	if err := MoveRollBack(g, rbTestID, reason, now)(rec); err != nil || rec.Move.Rollback.Reason != RollbackConfigChanged || rec.Move.Reason != RollbackConfigChanged || rec.Move.StuckReason != "" {
912	t.Fatalf("marked, caller says %q: %v %+v", reason, err, rec.Move.Rollback)
913	}
914	}
915	// A marked landing still refuses free text from the caller.
916	marked := rollbackTestLanding(g, now, LandConfigured)
917	if err := MoveMarkRollback(g, RollbackConfigChanged)(marked); err != nil {
918	t.Fatal(err)
919	}
920	if err := MoveRollBack(g, rbTestID, "declared lost: box-7", now)(marked); err == nil || marked.Move.State != MoveLanding || marked.Move.Rollback != nil || marked.Move.Reason != RollbackConfigChanged {
921	t.Fatalf("marked, free text: %v %+v", err, marked.Move)
922	}
923	// Every recorded rollback reason is taken; the declared loss is one.
924	for _, reason := range []string{RollbackConfigChanged, RollbackLandingDeadline, RollbackClaimWired, RollbackLandingLayout, RollbackInstallMismatch, RollbackFilesOccupied, RollbackProbeAttempts, RollbackCommitDeadline, RollbackDeclaredLost} {
925	rec := rollbackTestLanding(g, now, LandChecking)
926	if err := MoveRollBack(g, rbTestID, reason, now)(rec); err != nil || rec.Move.Rollback.Reason != reason {
927	t.Fatalf("%q: %v", reason, err)
928	}
929	}
930	// Refusals that write nothing: no reason on an unmarked landing, an
931	// unrecorded reason, a malformed rollback id, a zero time.
932	for name, step := range map[string]func(*Record) error{
933	"no reason": MoveRollBack(g, rbTestID, "", now),
934	"free text": MoveRollBack(g, rbTestID, "declared lost: box-7", now),
935	"move id": MoveRollBack(g, g.ID, RollbackLandingDeadline, now),
936	"short id": MoveRollBack(g, "rb_short", RollbackLandingDeadline, now),
937	"zero time": MoveRollBack(g, rbTestID, RollbackLandingDeadline, time.Time{}),
938	"another guard": MoveRollBack(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, rbTestID, RollbackLandingDeadline, now),
939	} {
940	rec := rollbackTestLanding(g, now, LandChecking)
941	if err := step(rec); err == nil || rec.Move.State != MoveLanding || rec.Region != "uk" || rec.Move.Rollback != nil {
942	t.Fatalf("%s: %v %+v", name, err, rec.Move)
943	}
944	}
945	// Only a landing rolls back: a replay finds rolling_back and is a
946	// conflict, which the caller answers by re-reading.
947	for _, state := range []string{MoveReady, MoveCutting, MoveReverting, MoveRollingBack, MoveStuck, MoveCommitted} {
948	rec := rollbackTestLanding(g, now, LandChecking)
949	rec.Move.State = state
950	if err := MoveRollBack(g, rbTestID, RollbackLandingDeadline, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Region != "uk" || rec.Move.Rollback != nil {
951	t.Fatalf("from %s: %v", state, err)
952	}
953	}
954	}
955
956	func TestRollbackTargetPhasesInOrder(t *testing.T) {
957	g := cutTestGuard()
958	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
959	step := func(rec *Record, from, to string, stamp func(*MoveRollback)) error {
960	return MoveRollbackTargetPhase(g, from, to, stamp)(rec)
961	}
962	recovered := func(rb *MoveRollback) { at := now; rb.RecoveredAt = &at }
963
964	// Dirty: every phase, in order; cleaning waits for the source's
965	// recovery (Review Focus 4).
966	rec := rollingBackRecord(t, g, now)
967	if err := step(rec, RbTargetReleasing, RbTargetReturning, nil); !errors.Is(err, ErrMoveConflict) {
968	t.Fatalf("skip released: %v", err)
969	}
970	if err := step(rec, RbTargetReleasing, RbTargetReleased, func(rb *MoveRollback) { rb.Dirty, rb.Evidence = true, "stopped-target.json" }); err != nil || !rec.Move.Rollback.Dirty {
971	t.Fatal(err)
972	}
973	if err := step(rec, RbTargetReleased, RbTargetCleaning, nil); !errors.Is(err, ErrMoveConflict) || rec.Move.Rollback.TargetPhase != RbTargetReleased {
974	t.Fatalf("a dirty release skipped the return: %v", err)
975	}
976	for _, s := range [][2]string{{RbTargetReleased, RbTargetReturning}, {RbTargetReturning, RbTargetReturned}} {
977	if err := step(rec, s[0], s[1], nil); err != nil {
978	t.Fatalf("%s to %s: %v", s[0], s[1], err)
979	}
980	}
981	if err := step(rec, RbTargetReturned, RbTargetCleaning, nil); !errors.Is(err, ErrMoveConflict) || rec.Move.Rollback.TargetPhase != RbTargetReturned {
982	t.Fatalf("cleaning before the source recovered: %v", err)
983	}
984	recovered(rec.Move.Rollback)
985	if err := step(rec, RbTargetReturned, RbTargetCleaning, nil); err != nil {
986	t.Fatal(err)
987	}
988	if err := step(rec, RbTargetCleaning, RbTargetCleaned, func(rb *MoveRollback) { at := now; rb.TargetParkedAt = &at }); err != nil || rec.Move.Rollback.TargetParkedAt == nil {
989	t.Fatal(err)
990	}
991	if err := step(rec, RbTargetCleaned, RbTargetReleasing, nil); !errors.Is(err, ErrMoveConflict) {
992	t.Fatalf("past cleaned: %v", err)
993	}
994
995	// Clean: released goes straight to cleaning and never returns.
996	rec = rollingBackRecord(t, g, now)
997	if err := step(rec, RbTargetReleasing, RbTargetReleased, nil); err != nil {
998	t.Fatal(err)
999	}
1000	if err := step(rec, RbTargetReleased, RbTargetReturning, nil); !errors.Is(err, ErrMoveConflict) {
1001	t.Fatalf("a clean release returned: %v", err)
1002	}
1003	recovered(rec.Move.Rollback)
1004	if err := step(rec, RbTargetReleased, RbTargetCleaning, nil); err != nil || rec.Move.Rollback.TargetPhase != RbTargetCleaning {
1005	t.Fatalf("the clean skip: %v", err)
1006	}
1007
1008	// The wrong from, the wrong guard, a stuck record, a landing: refused.
1009	rec = rollingBackRecord(t, g, now)
1010	if err := step(rec, RbTargetReleased, RbTargetReturning, nil); !errors.Is(err, ErrMoveConflict) {
1011	t.Fatalf("wrong from: %v", err)
1012	}
1013	if err := MoveRollbackTargetPhase(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, RbTargetReleasing, RbTargetReleased, nil)(rec); !errors.Is(err, ErrMoveConflict) {
1014	t.Fatalf("another guard: %v", err)
1015	}
1016	rec.Move.State = MoveStuck
1017	if err := step(rec, RbTargetReleasing, RbTargetReleased, nil); !errors.Is(err, ErrMoveConflict) {
1018	t.Fatalf("stuck: %v", err)
1019	}
1020	if err := step(rollbackTestLanding(g, now, LandProbing), RbTargetReleasing, RbTargetReleased, nil); !errors.Is(err, ErrMoveConflict) {
1021	t.Fatalf("landing: %v", err)
1022	}
1023	}
1024
1025	func TestRollbackTargetLostAndTheLossCommit(t *testing.T) {
1026	g := cutTestGuard()
1027	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
1028	const ref = "control/losses/a.go.example.com/loss_1.json"
1029	at := func(phase string, dirty bool) *Record {
1030	rec := rollingBackRecord(t, g, now)
1031	rec.Move.Rollback.TargetPhase, rec.Move.Rollback.Dirty = phase, dirty
1032	return rec
1033	}
1034	for _, c := range []struct {
1035	phase string
1036	dirty bool
1037	}{{RbTargetReleasing, false}, {RbTargetReturning, true}, {RbTargetReleased, true}} {
1038	rec := at(c.phase, c.dirty)
1039	if err := MoveRollbackTargetLost(g, ref, now)(rec); err != nil {
1040	t.Fatalf("from %s: %v", c.phase, err)
1041	}
1042	rb := rec.Move.Rollback
1043	if rb.TargetPhase != RbTargetLost || rb.LossRef != ref || rb.LostAt == nil || !rb.LostAt.Equal(now) || rb.Dirty != c.dirty || rec.Move.State != MoveRollingBack {
1044	t.Fatalf("from %s: %+v", c.phase, rb)
1045	}
1046	// A replay of the same declaration changes nothing; another is refused.
1047	if err := MoveRollbackTargetLost(g, ref, now.Add(time.Minute))(rec); err != nil || !rb.LostAt.Equal(now) {
1048	t.Fatalf("replay from %s: %v", c.phase, err)
1049	}
1050	if err := MoveRollbackTargetLost(g, ref+"x", now)(rec); !errors.Is(err, ErrMoveConflict) || rb.LossRef != ref {
1051	t.Fatalf("another declaration from %s: %v", c.phase, err)
1052	}
1053	// The forward order never leaves lost, and the source never takes it.
1054	for _, to := range []string{RbTargetReleased, RbTargetReturning, RbTargetReturned, RbTargetCleaning} {
1055	if err := MoveRollbackTargetPhase(g, RbTargetLost, to, nil)(rec); !errors.Is(err, ErrMoveConflict) {
1056	t.Fatalf("lost to %s: %v", to, err)
1057	}
1058	}
1059	if err := MoveRollbackSourcePhase(g, RbSourceWaiting, RbSourceClaimed, nil)(rec); !errors.Is(err, ErrMoveConflict) {
1060	t.Fatalf("the source consumed lost: %v", err)
1061	}
1062	// The loss commit: lost to released, clean, evidence the loss record.
1063	if err := MoveRollbackLossCommitted(g)(rec); err != nil || rb.TargetPhase != RbTargetReleased || rb.Dirty || rb.Evidence != ref || rb.LossRef != ref {
1064	t.Fatalf("loss commit from %s: %v %+v", c.phase, err, rb)
1065	}
1066	if err := MoveRollbackLossCommitted(g)(rec); err != nil || rb.TargetPhase != RbTargetReleased {
1067	t.Fatalf("loss commit replay: %v", err)
1068	}
1069	if err := MoveRollbackSourcePhase(g, RbSourceWaiting, RbSourceClaimed, nil)(rec); err != nil {
1070	t.Fatalf("the source after the loss commit: %v", err)
1071	}
1072	}
1073	// Refused: released clean (nothing left to declare), returned, cleaning,
1074	// cleaned; and the loss commit from anything but lost.
1075	for _, c := range []struct {
1076	phase string
1077	dirty bool
1078	}{{RbTargetReleased, false}, {RbTargetReturned, true}, {RbTargetCleaning, true}, {RbTargetCleaned, false}} {
1079	rec := at(c.phase, c.dirty)
1080	err := MoveRollbackTargetLost(g, ref, now)(rec)
1081	if !errors.Is(err, ErrMoveConflict) || rec.Move.Rollback.TargetPhase != c.phase || rec.Move.Rollback.LossRef != "" {
1082	t.Fatalf("lost from %s: %v", c.phase, err)
1083	}
1084	if !strings.HasSuffix(err.Error(), ": "+RefusedNothingToDeclare) {
1085	t.Fatalf("lost from %s: %v", c.phase, err)
1086	}
1087	}
1088	for _, phase := range []string{RbTargetReleasing, RbTargetReleased, RbTargetReturning, RbTargetReturned} {
1089	rec := at(phase, true)
1090	if err := MoveRollbackLossCommitted(g)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Rollback.TargetPhase != phase || !rec.Move.Rollback.Dirty {
1091	t.Fatalf("loss commit from %s: %v", phase, err)
1092	}
1093	}
1094	rec := at(RbTargetReleasing, false)
1095	rec.Move.State = MoveStuck
1096	if err := MoveRollbackTargetLost(g, ref, now)(rec); !errors.Is(err, ErrMoveConflict) {
1097	t.Fatalf("stuck: %v", err)
1098	}
1099	if err := MoveRollbackTargetLost(g, "", now)(at(RbTargetReleasing, false)); err == nil {
1100	t.Fatal("an empty loss reference")
1101	}
1102	}
1103
1104	func TestRollbackSourcePhasesCleanAndDirty(t *testing.T) {
1105	g := cutTestGuard()
1106	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
1107	walk := func(rec *Record, order []string) {
1108	t.Helper()
1109	for i := 0; i+1 < len(order); i++ {
1110	if err := MoveRollbackSourcePhase(g, order[i], order[i+1], nil)(rec); err != nil || rec.Move.Rollback.SourcePhase != order[i+1] {
1111	t.Fatalf("%s to %s: %v", order[i], order[i+1], err)
1112	}
1113	}
1114	}
1115	// waiting holds until the target's evidence: released clean, returned dirty.
1116	rec := rollingBackRecord(t, g, now)
1117	if err := MoveRollbackSourcePhase(g, RbSourceWaiting, RbSourceClaimed, nil)(rec); !errors.Is(err, ErrMoveConflict) {
1118	t.Fatalf("waiting while the target releases: %v", err)
1119	}
1120	rec.Move.Rollback.TargetPhase = RbTargetReleased
1121	walk(rec, []string{RbSourceWaiting, RbSourceClaimed, RbSourceRestoring})
1122	if err := MoveRollbackSourcePhase(g, RbSourceRestoring, RbSourceParking, nil)(rec); !errors.Is(err, ErrMoveConflict) {
1123	t.Fatalf("a clean return installed: %v", err)
1124	}
1125	if err := MoveRollbackSourcePhase(g, RbSourceRestoring, RbSourceProbing, nil)(rec); !errors.Is(err, ErrMoveConflict) {
1126	t.Fatalf("a clean return skipped verification: %v", err)
1127	}
1128	walk(rec, []string{RbSourceRestoring, RbSourceVerified, RbSourceProbing, RbSourceProbed})
1129	if err := MoveRollbackSourcePhase(g, RbSourceProbed, RbSourceRecovered, func(rb *MoveRollback) { at := now; rb.RecoveredAt = &at })(rec); err != nil || rec.Move.Rollback.RecoveredAt == nil {
1130	t.Fatal(err)
1131	}
1132	walk(rec, []string{RbSourceRecovered, RbSourceOpened})
1133	if err := MoveRollbackSourcePhase(g, RbSourceOpened, RbSourceWaiting, nil)(rec); !errors.Is(err, ErrMoveConflict) {
1134	t.Fatalf("past opened: %v", err)
1135	}
1136
1137	// Dirty: a released dirty target is not yet evidence; returned is.
1138	rec = rollingBackRecord(t, g, now)
1139	rec.Move.Rollback.TargetPhase, rec.Move.Rollback.Dirty = RbTargetReleased, true
1140	if err := MoveRollbackSourcePhase(g, RbSourceWaiting, RbSourceClaimed, nil)(rec); !errors.Is(err, ErrMoveConflict) {
1141	t.Fatalf("waiting while the target has not returned: %v", err)
1142	}
1143	rec.Move.Rollback.TargetPhase = RbTargetReturned
1144	walk(rec, []string{RbSourceWaiting, RbSourceClaimed, RbSourceRestoring})
1145	if err := MoveRollbackSourcePhase(g, RbSourceRestoring, RbSourceVerified, nil)(rec); !errors.Is(err, ErrMoveConflict) {
1146	t.Fatalf("a dirty return verified instead of installing: %v", err)
1147	}
1148	walk(rec, []string{RbSourceRestoring, RbSourceParking, RbSourceBuilding, RbSourcePublished, RbSourceLocal, RbSourceConfigured, RbSourceRestored, RbSourceProbing, RbSourceProbed, RbSourceRecovered, RbSourceOpened})
1149
1150	// Skips, the wrong guard and a stuck record are refused.
1151	rec = rollingBackRecord(t, g, now)
1152	rec.Move.Rollback.TargetPhase = RbTargetReleased
1153	if err := MoveRollbackSourcePhase(g, RbSourceWaiting, RbSourceRestoring, nil)(rec); !errors.Is(err, ErrMoveConflict) {
1154	t.Fatalf("skip claimed: %v", err)
1155	}
1156	if err := MoveRollbackSourcePhase(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, RbSourceWaiting, RbSourceClaimed, nil)(rec); !errors.Is(err, ErrMoveConflict) {
1157	t.Fatalf("another guard: %v", err)
1158	}
1159	rec.Move.State = MoveStuck
1160	if err := MoveRollbackSourcePhase(g, RbSourceWaiting, RbSourceClaimed, nil)(rec); !errors.Is(err, ErrMoveConflict) {
1161	t.Fatalf("stuck: %v", err)
1162	}
1163	}
1164
1165	func TestMoveRetryEpochRefusals(t *testing.T) {
1166	for _, rollback := range []bool{false, true} {
1167	for _, epoch := range []uint64{0, 7, ^uint64(0)} {
1168	g := cutTestGuard()
1169	rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation,
1170	State: MoveStuck, StuckReason: StuckRecoveryProbeAttempts, RecoveryRetryEpoch: epoch}}
1171	if rollback {
1172	rec.Move.Rollback = &MoveRollback{ID: rbTestID, StuckAt: RbSourceProbing}
1173	}
1174	before, _ := json.Marshal(rec)
1175	err := MoveRetryRecovery(g)(rec)
1176	if epoch == ^uint64(0) {
1177	after, _ := json.Marshal(rec)
1178	if !errors.Is(err, ErrMoveConflict) || string(before) != string(after) {
1179	t.Fatalf("epoch overflow mutated record: %v %+v", err, rec.Move)
1180	}
1181	continue
1182	}
1183	if err != nil || rec.Move.RecoveryRetryEpoch != epoch+1 {
1184	t.Fatalf("retry epoch: %v %+v", err, rec.Move)
1185	}
1186	before, _ = json.Marshal(rec)
1187	err = MoveRetryRecovery(g)(rec)
1188	after, _ := json.Marshal(rec)
1189	if !errors.Is(err, ErrMoveConflict) || string(before) != string(after) {
1190	t.Fatalf("retry replay mutated record: %v %+v", err, rec.Move)
1191	}
1192	}
1193	}
1194	}
1195
1196	// The legacy retry (flag off) must write the same bytes it always did: it
1197	// never touches the recovery epoch, set or not.
1198	func TestLegacyMoveRetryLeavesTheEpochAlone(t *testing.T) {
1199	for _, rollback := range []bool{false, true} {
1200	for _, epoch := range []uint64{0, 7, ^uint64(0)} {
1201	g := cutTestGuard()
1202	rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation,
1203	State: MoveStuck, StuckReason: StuckRecoveryProbeAttempts, RecoveryRetryEpoch: epoch}}
1204	if rollback {
1205	rec.Move.Rollback = &MoveRollback{ID: rbTestID, StuckAt: RbSourceProbing}
1206	}
1207	if err := MoveRetry(g)(rec); err != nil || rec.Move.RecoveryRetryEpoch != epoch {
1208	t.Fatalf("legacy retry: %v epoch %d, want %d", err, rec.Move.RecoveryRetryEpoch, epoch)
1209	}
1210	raw, _ := json.Marshal(rec.Move)
1211	if epoch == 0 && strings.Contains(string(raw), "recovery_retry_epoch") {
1212	t.Fatalf("legacy retry wrote the epoch: %s", raw)
1213	}
1214	}
1215	}
1216	}
1217
1218	func TestStuckAndRetryReturnToThePhase(t *testing.T) {
1219	g := cutTestGuard()
1220	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
1221	at := func(phase string) *Record {
1222	rec := rollingBackRecord(t, g, now)
1223	rec.Move.Rollback.TargetPhase, rec.Move.Rollback.SourcePhase = RbTargetReleased, phase
1224	return rec
1225	}
1226	for _, c := range []struct{ phase, reason string }{
1227	{RbSourceRestoring, StuckBaselineMismatch},
1228	{RbSourceVerified, StuckBaselineMismatch},
1229	{RbSourceProbing, StuckRecoveryProbeAttempts},
1230	} {
1231	rec := at(c.phase)
1232	if err := MoveMarkStuck(g, c.reason)(rec); err != nil {
1233	t.Fatalf("%s: %v", c.phase, err)
1234	}
1235	m, rb := rec.Move, rec.Move.Rollback
1236	if m.State != MoveStuck || m.StuckReason != c.reason || m.Reason != "" || rb.StuckAt != c.phase || rb.SourcePhase != c.phase || rb.Reason != RollbackProbeAttempts {
1237	t.Fatalf("%s: %+v %+v", c.phase, m, rb)
1238	}
1239	// The same mark again is a replay; another is refused.
1240	if err := MoveMarkStuck(g, c.reason)(rec); err != nil {
1241	t.Fatalf("%s replay: %v", c.phase, err)
1242	}
1243	other := StuckBaselineMismatch
1244	if c.reason == other {
1245	other = StuckRecoveryProbeAttempts
1246	}
1247	if err := MoveMarkStuck(g, other)(rec); !errors.Is(err, ErrMoveConflict) || m.StuckReason != c.reason {
1248	t.Fatalf("%s another reason: %v", c.phase, err)
1249	}
1250	// Nothing advances a stuck source.
1251	if err := MoveRollbackSourcePhase(g, c.phase, RbSourceProbed, nil)(rec); !errors.Is(err, ErrMoveConflict) {
1252	t.Fatalf("%s advanced while stuck: %v", c.phase, err)
1253	}
1254	// Retry: back to rolling_back at the phase it stuck in, so a
1255	// mismatch verifies again; the stuck reason goes, the rollback's stays.
1256	if err := MoveRetry(g)(rec); err != nil {
1257	t.Fatalf("%s retry: %v", c.phase, err)
1258	}
1259	if m.State != MoveRollingBack || m.StuckReason != "" || m.Reason != "" || rb.SourcePhase != c.phase || rb.StuckAt != "" || rb.Reason != RollbackProbeAttempts {
1260	t.Fatalf("%s after retry: %+v %+v", c.phase, m, rb)
1261	}
1262	if err := MoveRetry(g)(rec); !errors.Is(err, ErrMoveConflict) || !strings.HasSuffix(err.Error(), ": "+RefusedNotStuck) {
1263	t.Fatalf("%s retry twice: %v", c.phase, err)
1264	}
1265	}
1266	// Each stuck reason is the phase's own, and only these phases stick.
1267	for _, c := range []struct{ phase, reason string }{
1268	{RbSourceProbing, StuckBaselineMismatch},
1269	{RbSourceVerified, StuckRecoveryProbeAttempts},
1270	{RbSourceVerified, StuckRevertProbeBudget},
1271	{RbSourceWaiting, StuckBaselineMismatch},
1272	{RbSourceClaimed, StuckBaselineMismatch},
1273	{RbSourceParking, StuckBaselineMismatch},
1274	{RbSourceProbed, StuckRecoveryProbeAttempts},
1275	{RbSourceOpened, StuckRecoveryProbeAttempts},
1276	} {
1277	rec := at(c.phase)
1278	if err := MoveMarkStuck(g, c.reason)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveRollingBack || rec.Move.Rollback.StuckAt != "" {
1279	t.Fatalf("%s with %q: %v", c.phase, c.reason, err)
1280	}
1281	}
1282	if err := MoveMarkStuck(g, "the probe failed: connection refused")(at(RbSourceProbing)); err == nil || errors.Is(err, ErrMoveConflict) {
1283	t.Fatalf("free text: %v", err)
1284	}
1285
1286	// A marked rollback keeps its cause, on the move and on the journal,
1287	// through a stuck and a retry: only the stuck reason comes and goes.
1288	marked := rollbackTestLanding(g, now, LandProbing)
1289	if err := MoveMarkRollback(g, RollbackProbeAttempts)(marked); err != nil {
1290	t.Fatal(err)
1291	}
1292	if err := MoveRollBack(g, rbTestID, "", now)(marked); err != nil {
1293	t.Fatal(err)
1294	}
1295	marked.Move.Rollback.TargetPhase, marked.Move.Rollback.SourcePhase = RbTargetReleased, RbSourceProbing
1296	if err := MoveMarkStuck(g, StuckRecoveryProbeAttempts)(marked); err != nil || marked.Move.StuckReason != StuckRecoveryProbeAttempts || marked.Move.Reason != RollbackProbeAttempts || marked.Move.Rollback.Reason != RollbackProbeAttempts {
1297	t.Fatalf("marked rollback stuck: %v %+v", err, marked.Move)
1298	}
1299	if err := MoveRetry(g)(marked); err != nil || marked.Move.StuckReason != "" || marked.Move.Reason != RollbackProbeAttempts || marked.Move.Rollback.Reason != RollbackProbeAttempts {
1300	t.Fatalf("marked rollback retry: %v %+v", err, marked.Move)
1301	}
1302
1303	// A revert sticks with no rollback journal and retries to reverting; its
1304	// own cause survives both.
1305	rec := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveReverting, CutPhase: CutStaged, Reason: RevertCutDeadline}}
1306	// A revert's probe has the recovery probe's nine attempts as well as its
1307	// fifteen minutes (§7.7; Task 6 fix round 1), so both reasons are a
1308	// revert's.
1309	for _, reason := range []string{StuckRevertProbeBudget, StuckBaselineMismatch, StuckRecoveryProbeAttempts} {
1310	if err := MoveMarkStuck(g, reason)(rec); err != nil || rec.Move.State != MoveStuck || rec.Move.StuckReason != reason || rec.Move.Reason != RevertCutDeadline || rec.Move.Rollback != nil {
1311	t.Fatalf("revert %q: %v %+v", reason, err, rec.Move)
1312	}
1313	if err := MoveRetry(g)(rec); err != nil || rec.Move.State != MoveReverting || rec.Move.StuckReason != "" || rec.Move.Reason != RevertCutDeadline || rec.Move.Rollback != nil || rec.Move.CutPhase != CutStaged {
1314	t.Fatalf("revert retry: %v %+v", err, rec.Move)
1315	}
1316	}
1317	// Neither a landing nor a committed move sticks or retries.
1318	for _, state := range []string{MoveLanding, MoveCommitted, MoveCutting} {
1319	rec := rollbackTestLanding(g, now, LandProbing)
1320	rec.Move.State = state
1321	if err := MoveMarkStuck(g, StuckRecoveryProbeAttempts)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != state {
1322	t.Fatalf("stuck from %s: %v", state, err)
1323	}
1324	if err := MoveRetry(g)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != state {
1325	t.Fatalf("retry from %s: %v", state, err)
1326	}
1327	}
1328	// A stuck rollback that records no phase is not retried blind.
1329	rec = at(RbSourceProbing)
1330	rec.Move.State, rec.Move.StuckReason = MoveStuck, StuckRecoveryProbeAttempts
1331	if err := MoveRetry(g)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveStuck {
1332	t.Fatalf("retry with no stuck phase: %v", err)
1333	}
1334	}
1335
1336	// The dirty return's install (Plan 4 Task 3, ruling J13) refuses into stuck,
1337	// never into a landing's rollback: the files paths still occupied after the
1338	// source's own pre-move files were parked stop it at local, and a build that
1339	// does not verify stops it at configured (where the database is installed).
1340	// Each reason is its phase's own, for a dirty return only, and a retry goes
1341	// back to that phase.
1342	func TestStuckReturnInstallReasons(t *testing.T) {
1343	g := cutTestGuard()
1344	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
1345	at := func(phase string, dirty bool) *Record {
1346	rec := rollingBackRecord(t, g, now)
1347	rec.Move.Rollback.TargetPhase, rec.Move.Rollback.SourcePhase, rec.Move.Rollback.Dirty = RbTargetReturned, phase, dirty
1348	return rec
1349	}
1350	for _, c := range []struct{ phase, reason string }{
1351	{RbSourceLocal, StuckReturnFilesOccupied},
1352	{RbSourceConfigured, StuckReturnInstallMismatch},
1353	{RbSourceLocal, StuckReturnUnknownStamp},
1354	} {
1355	rec := at(c.phase, true)
1356	if err := MoveMarkStuck(g, c.reason)(rec); err != nil {
1357	t.Fatalf("%s: %v", c.phase, err)
1358	}
1359	m, rb := rec.Move, rec.Move.Rollback
1360	if m.State != MoveStuck || m.StuckReason != c.reason || rb.StuckAt != c.phase || rb.Reason != RollbackProbeAttempts {
1361	t.Fatalf("%s: %+v %+v", c.phase, m, rb)
1362	}
1363	if err := MoveMarkStuck(g, c.reason)(rec); err != nil {
1364	t.Fatalf("%s replay: %v", c.phase, err)
1365	}
1366	if err := MoveRetry(g)(rec); err != nil || m.State != MoveRollingBack || rb.SourcePhase != c.phase || m.StuckReason != "" {
1367	t.Fatalf("%s retry: %v %+v %+v", c.phase, err, m, rb)
1368	}
1369	}
1370	for _, c := range []struct {
1371	phase, reason string
1372	dirty bool
1373	}{
1374	{RbSourceLocal, StuckReturnFilesOccupied, false},
1375	{RbSourceConfigured, StuckReturnInstallMismatch, false},
1376	{RbSourceConfigured, StuckReturnFilesOccupied, true},
1377	{RbSourceLocal, StuckReturnInstallMismatch, true},
1378	{RbSourceParking, StuckReturnFilesOccupied, true},
1379	{RbSourceRestored, StuckReturnInstallMismatch, true},
1380	{RbSourceVerified, StuckReturnFilesOccupied, false},
1381	{RbSourceLocal, StuckReturnUnknownStamp, false},
1382	{RbSourceConfigured, StuckReturnUnknownStamp, true},
1383	{RbSourceParking, StuckReturnUnknownStamp, true},
1384	} {
1385	rec := at(c.phase, c.dirty)
1386	if err := MoveMarkStuck(g, c.reason)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveRollingBack {
1387	t.Fatalf("%s (dirty %v) with %q: %v", c.phase, c.dirty, c.reason, err)
1388	}
1389	}
1390	// A revert installs nothing, so neither reason is a revert's.
1391	rev := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveReverting, CutPhase: CutStaged, Reason: RevertCutDeadline}}
1392	for _, reason := range []string{StuckReturnFilesOccupied, StuckReturnInstallMismatch, StuckReturnUnknownStamp} {
1393	if err := MoveMarkStuck(g, reason)(rev); !errors.Is(err, ErrMoveConflict) || rev.Move.State != MoveReverting {
1394	t.Fatalf("a revert with %q: %v", reason, err)
1395	}
1396	}
1397	}
1398
1399	// A dirty return the target cannot build (Task 4 fix round 1, ruling 2)
1400	// stops for an operator: the database is missing, the target changed after
1401	// it stopped, or its files cannot be archived. Each reason is the target's
1402	// returning phase's own, on a dirty rollback; the target phase and Dirty
1403	// stay, the source's phase (waiting) is StuckAt, and a retry goes back to
1404	// it, so the target runs its return again.
1405	func TestStuckReturnTargetReasons(t *testing.T) {
1406	g := cutTestGuard()
1407	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
1408	at := func(target string, dirty bool) *Record {
1409	rec := rollingBackRecord(t, g, now)
1410	rec.Move.Rollback.TargetPhase, rec.Move.Rollback.Dirty = target, dirty
1411	return rec
1412	}
1413	reasons := []string{StuckReturnTargetDatabase, StuckReturnTargetChanged, StuckReturnTargetFiles}
1414	for _, reason := range reasons {
1415	rec := at(RbTargetReturning, true)
1416	if err := MoveMarkStuck(g, reason)(rec); err != nil {
1417	t.Fatalf("%q: %v", reason, err)
1418	}
1419	m, rb := rec.Move, rec.Move.Rollback
1420	if m.State != MoveStuck || m.StuckReason != reason || rb.StuckAt != RbSourceWaiting || rb.TargetPhase != RbTargetReturning || !rb.Dirty {
1421	t.Fatalf("%q: %+v %+v", reason, m, rb)
1422	}
1423	if err := MoveMarkStuck(g, reason)(rec); err != nil {
1424	t.Fatalf("%q replay: %v", reason, err)
1425	}
1426	if err := MoveRetry(g)(rec); err != nil || m.State != MoveRollingBack || rb.SourcePhase != RbSourceWaiting || rb.TargetPhase != RbTargetReturning || m.StuckReason != "" {
1427	t.Fatalf("%q retry: %v %+v %+v", reason, err, m, rb)
1428	}
1429	}
1430	for _, reason := range reasons {
1431	for _, c := range []struct {
1432	target string
1433	dirty bool
1434	}{{RbTargetReturning, false}, {RbTargetReleasing, true}, {RbTargetReleased, true}, {RbTargetReturned, true}, {RbTargetCleaning, true}} {
1435	rec := at(c.target, c.dirty)
1436	if err := MoveMarkStuck(g, reason)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.State != MoveRollingBack {
1437	t.Fatalf("%q at %s (dirty %v): %v", reason, c.target, c.dirty, err)
1438	}
1439	}
1440	// The source's own install phases are not the target's.
1441	rec := at(RbTargetReturned, true)
1442	rec.Move.Rollback.SourcePhase = RbSourceLocal
1443	if err := MoveMarkStuck(g, reason)(rec); !errors.Is(err, ErrMoveConflict) {
1444	t.Fatalf("%q at the source's local: %v", reason, err)
1445	}
1446	rev := &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveReverting, CutPhase: CutStaged, Reason: RevertCutDeadline}}
1447	if err := MoveMarkStuck(g, reason)(rev); !errors.Is(err, ErrMoveConflict) {
1448	t.Fatalf("a revert with %q: %v", reason, err)
1449	}
1450	}
1451	}
1452
1453	// TestStuckReleaseTargetFiles (Plan 5 Task 9a): the target's release finds
1454	// files it can never archive. The reason belongs to the target's releasing,
1455	// with the source still waiting, dirty decided or not; the target phase
1456	// stays releasing, the source's waiting is StuckAt, and a retry runs the
1457	// release again. Anywhere else it is a conflict.
1458	func TestStuckReleaseTargetFiles(t *testing.T) {
1459	g := cutTestGuard()
1460	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
1461	at := func(target string, dirty bool) *Record {
1462	rec := rollingBackRecord(t, g, now)
1463	rec.Move.Rollback.TargetPhase, rec.Move.Rollback.Dirty = target, dirty
1464	return rec
1465	}
1466	for _, dirty := range []bool{false, true} {
1467	rec := at(RbTargetReleasing, dirty)
1468	if err := MoveMarkStuck(g, StuckReleaseTargetFiles)(rec); err != nil {
1469	t.Fatalf("dirty %v: %v", dirty, err)
1470	}
1471	m, rb := rec.Move, rec.Move.Rollback
1472	if m.State != MoveStuck || m.StuckReason != StuckReleaseTargetFiles || rb.StuckAt != RbSourceWaiting || rb.TargetPhase != RbTargetReleasing {
1473	t.Fatalf("dirty %v: %+v %+v", dirty, m, rb)
1474	}
1475	if err := MoveMarkStuck(g, StuckReleaseTargetFiles)(rec); err != nil {
1476	t.Fatalf("replay: %v", err)
1477	}
1478	if err := MoveRetry(g)(rec); err != nil || m.State != MoveRollingBack || rb.SourcePhase != RbSourceWaiting || rb.TargetPhase != RbTargetReleasing || m.StuckReason != "" {
1479	t.Fatalf("retry: %v %+v %+v", err, m, rb)
1480	}
1481	}
1482	for _, target := range []string{RbTargetReleased, RbTargetReturning, RbTargetReturned, RbTargetCleaning} {
1483	if err := MoveMarkStuck(g, StuckReleaseTargetFiles)(at(target, true)); !errors.Is(err, ErrMoveConflict) {
1484	t.Fatalf("at %s: %v", target, err)
1485	}
1486	}
1487	rec := at(RbTargetReleasing, false)
1488	rec.Move.Rollback.SourcePhase = RbSourceClaimed
1489	if err := MoveMarkStuck(g, StuckReleaseTargetFiles)(rec); !errors.Is(err, ErrMoveConflict) {
1490	t.Fatalf("with the source claimed: %v", err)
1491	}
1492	if StuckReleaseTargetFiles == StuckReturnTargetFiles {
1493	t.Fatal("the release's reason must differ from the return's")
1494	}
1495	}
1496
1497	func TestRolledBackCloseNeedsAllThreeStamps(t *testing.T) {
1498	g := cutTestGuard()
1499	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
1500	later := now.Add(3 * time.Minute)
1501	opened := now.Add(50 * time.Second)
1502	full := func() *Record {
1503	rec := rollingBackRecord(t, g, now)
1504	rb := rec.Move.Rollback
1505	rb.TargetPhase, rb.SourcePhase = RbTargetCleaned, RbSourceOpened
1506	rb.RecoveredAt, rb.OpenedAt, rb.TargetParkedAt = &now, &opened, &now
1507	return rec
1508	}
1509	for name, clear := range map[string]func(*MoveRollback){
1510	"recovered": func(rb *MoveRollback) { rb.RecoveredAt = nil },
1511	"opened": func(rb *MoveRollback) { rb.OpenedAt = nil },
1512	"target parked": func(rb *MoveRollback) { rb.TargetParkedAt = nil },
1513	} {
1514	rec := full()
1515	clear(rec.Move.Rollback)
1516	if err := MoveRolledBackClose(g, later)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil || rec.LastMove != nil {
1517	t.Fatalf("close without %s: %v", name, err)
1518	}
1519	}
1520	rec := full()
1521	rec.Move.State = MoveStuck
1522	if err := MoveRolledBackClose(g, later)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil {
1523	t.Fatalf("close a stuck record: %v", err)
1524	}
1525	rec = full()
1526	rec.Region = "uk"
1527	if err := MoveRolledBackClose(g, later)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil {
1528	t.Fatalf("close with the region not the source's: %v", err)
1529	}
1530	rec = full()
1531	if err := MoveRolledBackClose(g, later)(rec); err != nil || rec.Move != nil || rec.Region != "ie" {
1532	t.Fatalf("close: %v %+v", err, rec)
1533	}
1534	// OfflineS is the rollback's open minus the cut: 50 s + 10 s.
1535	if lm := rec.LastMove; lm == nil || lm.ID != g.ID || lm.Outcome != "rolled_back" || !lm.At.Equal(later) || lm.OfflineS != 60 {
1536	t.Fatalf("last move: %+v", rec.LastMove)
1537	}
1538	}
1539
1540	func TestRevertCloseCountsToTheReopen(t *testing.T) {
1541	g := cutTestGuard()
1542	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
1543	cut := now.Add(-30 * time.Second)
1544	reverting := func() *Record {
1545	return &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveReverting, CutPhase: CutStopped, CutAt: &cut, Reason: RevertCutDeadline}}
1546	}
1547	rec := reverting()
1548	if err := MoveRevertClose(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil {
1549	t.Fatalf("close before the reopen: %v", err)
1550	}
1551	if err := MoveRevertReopened(g, now)(rec); err != nil || rec.Move.ReopenedAt == nil || !rec.Move.ReopenedAt.Equal(now) {
1552	t.Fatalf("reopened: %v", err)
1553	}
1554	if err := MoveRevertReopened(g, now.Add(time.Minute))(rec); err != nil || !rec.Move.ReopenedAt.Equal(now) {
1555	t.Fatalf("reopened replay keeps the first stamp: %v", err)
1556	}
1557	if err := MoveRevertClose(g, now.Add(time.Minute))(rec); err != nil || rec.Move != nil || rec.Region != "ie" {
1558	t.Fatalf("revert close: %v %+v", err, rec)
1559	}
1560	if lm := rec.LastMove; lm == nil || lm.ID != g.ID || lm.Outcome != "rolled_back" || lm.OfflineS != 30 || !lm.At.Equal(now.Add(time.Minute)) {
1561	t.Fatalf("last move: %+v", rec.LastMove)
1562	}
1563	// Only a revert reopens or closes this way.
1564	for _, state := range []string{MoveCutting, MoveLanding, MoveRollingBack, MoveStuck, MoveCommitted} {
1565	rec := reverting()
1566	rec.Move.State, rec.Move.ReopenedAt = state, &now
1567	if err := MoveRevertClose(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil {
1568	t.Fatalf("revert close from %s: %v", state, err)
1569	}
1570	rec.Move.ReopenedAt = nil
1571	if err := MoveRevertReopened(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.ReopenedAt != nil {
1572	t.Fatalf("reopened from %s: %v", state, err)
1573	}
1574	}
1575	rec = reverting()
1576	rec.Move.ReopenedAt, rec.Region = &now, "uk"
1577	if err := MoveRevertClose(g, now)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil {
1578	t.Fatalf("revert close away from the source: %v", err)
1579	}
1580	}
1581
1582	// The new journal fields survive the record's own write path, under their
1583	// own keys, and stay off a record that does not use them.
1584	func TestRollbackJournalThroughTheRecordWrite(t *testing.T) {
1585	s := release.NewDirStore(t.TempDir())
1586	rec := moveTestRecord(t, s)
1587	g := cutTestGuard()
1588	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
1589	put := rollbackTestLanding(g, now, LandProbing)
1590	ctx := context.Background()
1591	if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, func(r *Record) error { r.Region, r.Move = put.Region, put.Move; return nil }); err != nil {
1592	t.Fatal(err)
1593	}
1594	const ref = "control/losses/a.go.example.com/loss_1.json"
1595	for _, step := range []func(*Record) error{
1596	MoveRollBack(g, rbTestID, RollbackDeclaredLost, now),
1597	MoveRollbackTargetLost(g, ref, now),
1598	} {
1599	if _, err := SetRecordMove(ctx, s, rec.Account, rec.App, rec.Host, step); err != nil {
1600	t.Fatal(err)
1601	}
1602	}
1603	got, err := GetRecord(ctx, s, rec.Account, rec.App, rec.Host)
1604	if err != nil {
1605	t.Fatal(err)
1606	}
1607	if got.Region != "ie" || got.Move.State != MoveRollingBack || got.Move.Rollback.ID != rbTestID || got.Move.Rollback.Reason != RollbackDeclaredLost || got.Move.Rollback.TargetPhase != RbTargetLost || got.Move.Rollback.LossRef != ref || got.Move.Rollback.LostAt == nil {
1608	t.Fatalf("read back: %+v %+v", got.Move, got.Move.Rollback)
1609	}
1610	raw, err := json.Marshal(MoveState{ReopenedAt: &now, StuckReason: StuckRecoveryProbeAttempts, Rollback: &MoveRollback{StuckAt: RbSourceProbing, LossRef: ref, LostAt: &now}})
1611	if err != nil {
1612	t.Fatal(err)
1613	}
1614	for _, key := range []string{`"reopened_at"`, `"stuck_reason":"the recovery probe ran out of attempts"`, `"stuck_at":"probing"`, `"loss_ref"`, `"lost_at"`} {
1615	if !strings.Contains(string(raw), key) {
1616	t.Fatalf("%s missing from %s", key, raw)
1617	}
1618	}
1619	if raw, _ := json.Marshal(MoveState{Rollback: &MoveRollback{}}); strings.Contains(string(raw), "stuck_at") || strings.Contains(string(raw), "stuck_reason") || strings.Contains(string(raw), "reopened_at") || strings.Contains(string(raw), "loss_ref") {
1620	t.Fatalf("the new keys are not omitempty: %s", raw)
1621	}
1622	}
1623
1624	func TestRollbackIDShape(t *testing.T) {
1625	for id, want := range map[string]bool{rbTestID: true, "mv_abcdefghijklmnopqrstuvwxyz": false, "rb_short": false, "rb_ABCDEFGHIJKLMNOPQRSTUVWXYZ": false, "": false} {
1626	if ValidRollbackID(id) != want {
1627	t.Fatalf("ValidRollbackID(%q) = %v", id, !want)
1628	}
1629	}
1630	}
1631
1632	// A recovery probe records the inputs it renders before it renders them
1633	// (Plan 4 Task 5; §7.7, J2-R2-5; Task 3b concern 3): MoveRecordRecoveryInputs
1634	// writes Rollback.RecoveryInputs on this rollback while its source is at
1635	// probing, and nothing else. It replaces what configured recorded (a bundle
1636	// published since is the probe's to render, never compared with
1637	// move.inputs), and is refused at any other phase, on another rollback, on
1638	// a stuck record and without a release.
1639	func TestRecoveryInputsRecordedAtProbing(t *testing.T) {
1640	g := cutTestGuard()
1641	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
1642	at := func(phase string, dirty bool) *Record {
1643	rec := rollingBackRecord(t, g, now)
1644	rec.Move.Rollback.TargetPhase, rec.Move.Rollback.SourcePhase, rec.Move.Rollback.Dirty = RbTargetReturned, phase, dirty
1645	rec.Move.Inputs = &MoveInputs{BundleETags: map[string]string{"default": "e1"}, Release: "v7"}
1646	return rec
1647	}
1648	first := MoveInputs{BundleETags: map[string]string{"default": "e2"}, OverlayETag: "o2", Release: "v8"}
1649	later := MoveInputs{BundleETags: map[string]string{"default": "e3"}, Release: "v8"}
1650	for _, dirty := range []bool{false, true} {
1651	rec := at(RbSourceProbing, dirty)
1652	if dirty {
1653	rec.Move.Rollback.RecoveryInputs = &MoveInputs{BundleETags: map[string]string{"default": "e2"}, Release: "v8"}
1654	}
1655	if err := MoveRecordRecoveryInputs(g, rbTestID, first)(rec); err != nil {
1656	t.Fatalf("dirty %v: %v", dirty, err)
1657	}
1658	rb := rec.Move.Rollback
1659	if rb.RecoveryInputs == nil || rb.RecoveryInputs.BundleETags["default"] != "e2" || rb.RecoveryInputs.OverlayETag != "o2" || rb.RecoveryInputs.Release != "v8" {
1660	t.Fatalf("dirty %v: recorded %+v", dirty, rb.RecoveryInputs)
1661	}
1662	if rec.Move.Inputs.BundleETags["default"] != "e1" || rec.Move.Inputs.Release != "v7" || rb.SourcePhase != RbSourceProbing || rec.Move.State != MoveRollingBack {
1663	t.Fatalf("dirty %v: the record changed beyond the inputs: %+v %+v", dirty, rec.Move, rb)
1664	}
1665	// The caller's map is copied, not aliased.
1666	first.BundleETags["default"] = "mutated"
1667	if rb.RecoveryInputs.BundleETags["default"] != "e2" {
1668	t.Fatalf("dirty %v: the record aliases the caller's map", dirty)
1669	}
1670	first.BundleETags["default"] = "e2"
1671	// A restarted probe records what it renders now.
1672	if err := MoveRecordRecoveryInputs(g, rbTestID, later)(rec); err != nil || rb.RecoveryInputs.BundleETags["default"] != "e3" {
1673	t.Fatalf("dirty %v: a later probe: %v %+v", dirty, err, rb.RecoveryInputs)
1674	}
1675	}
1676	for _, phase := range []string{RbSourceWaiting, RbSourceClaimed, RbSourceRestoring, RbSourceVerified, RbSourceConfigured, RbSourceRestored, RbSourceProbed, RbSourceRecovered, RbSourceOpened} {
1677	rec := at(phase, phase == RbSourceConfigured || phase == RbSourceRestored)
1678	if err := MoveRecordRecoveryInputs(g, rbTestID, first)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Rollback.RecoveryInputs != nil {
1679	t.Fatalf("at %s: %v %+v", phase, err, rec.Move.Rollback.RecoveryInputs)
1680	}
1681	}
1682	rec := at(RbSourceProbing, false)
1683	if err := MoveRecordRecoveryInputs(g, "rb_bbbbbbbbbbbbbbbbbbbbbbbbbb", first)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Rollback.RecoveryInputs != nil {
1684	t.Fatalf("another rollback: %v", err)
1685	}
1686	if err := MoveRecordRecoveryInputs(g, rbTestID, MoveInputs{BundleETags: map[string]string{"default": "e2"}})(rec); err == nil || rec.Move.Rollback.RecoveryInputs != nil {
1687	t.Fatalf("no release: %v", err)
1688	}
1689	if err := MoveRecordRecoveryInputs(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, rbTestID, first)(rec); !errors.Is(err, ErrMoveConflict) {
1690	t.Fatalf("another guard: %v", err)
1691	}
1692	rec.Move.State = MoveStuck
1693	if err := MoveRecordRecoveryInputs(g, rbTestID, first)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.Rollback.RecoveryInputs != nil {
1694	t.Fatalf("stuck: %v", err)
1695	}
1696	}
1697
1698	// A revert's probe records the inputs it renders before it renders them
1699	// (Plan 4 Task 6; J18, §7.7): MoveRecordRevertInputs writes the move's own
1700	// RecoveryInputs on a reverting record that has not reopened, and nothing
1701	// else. A later probe replaces them; they are never compared with
1702	// move.inputs. It is refused on any other state, after the reopen, without a
1703	// release and under another guard.
1704	func TestRevertInputsRecordedBeforeTheRender(t *testing.T) {
1705	g := cutTestGuard()
1706	now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
1707	reverting := func() *Record {
1708	return &Record{Region: "ie", Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, From: "ie", To: "uk", State: MoveReverting, CutPhase: CutStopping,
1709	Reason: RevertStopEvidenceBudget, Inputs: &MoveInputs{BundleETags: map[string]string{"default": "e1"}, Release: "v7"}}}
1710	}
1711	first := MoveInputs{BundleETags: map[string]string{"default": "e2"}, OverlayETag: "o2", Release: "v8"}
1712	rec := reverting()
1713	if err := MoveRecordRevertInputs(g, first)(rec); err != nil {
1714	t.Fatal(err)
1715	}
1716	m := rec.Move
1717	if m.RecoveryInputs == nil || m.RecoveryInputs.BundleETags["default"] != "e2" || m.RecoveryInputs.OverlayETag != "o2" || m.RecoveryInputs.Release != "v8" {
1718	t.Fatalf("recorded %+v", m.RecoveryInputs)
1719	}
1720	if m.Inputs.BundleETags["default"] != "e1" || m.Inputs.Release != "v7" || m.State != MoveReverting || m.Reason != RevertStopEvidenceBudget || m.CutPhase != CutStopping {
1721	t.Fatalf("the record changed beyond the inputs: %+v", m)
1722	}
1723	first.BundleETags["default"] = "mutated"
1724	if m.RecoveryInputs.BundleETags["default"] != "e2" {
1725	t.Fatal("the record aliases the caller's map")
1726	}
1727	if err := MoveRecordRevertInputs(g, MoveInputs{BundleETags: map[string]string{"default": "e3"}, Release: "v8"})(rec); err != nil || m.RecoveryInputs.BundleETags["default"] != "e3" {
1728	t.Fatalf("a later probe: %v %+v", err, m.RecoveryInputs)
1729	}
1730	for _, state := range []string{MoveCutting, MoveLanding, MoveRollingBack, MoveStuck, MoveCommitted, MoveAborting} {
1731	rec := reverting()
1732	rec.Move.State = state
1733	if err := MoveRecordRevertInputs(g, first)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.RecoveryInputs != nil {
1734	t.Fatalf("from %s: %v", state, err)
1735	}
1736	}
1737	rec = reverting()
1738	rec.Move.ReopenedAt = &now
1739	if err := MoveRecordRevertInputs(g, first)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.RecoveryInputs != nil {
1740	t.Fatalf("after the reopen: %v", err)
1741	}
1742	rec = reverting()
1743	if err := MoveRecordRevertInputs(g, MoveInputs{BundleETags: map[string]string{"default": "e2"}})(rec); err == nil || rec.Move.RecoveryInputs != nil {
1744	t.Fatalf("no release: %v", err)
1745	}
1746	if err := MoveRecordRevertInputs(MoveGuard{ID: g.ID, Incarnation: strings.Repeat("cd", 32)}, first)(rec); !errors.Is(err, ErrMoveConflict) || rec.Move.RecoveryInputs != nil {
1747	t.Fatalf("another guard: %v", err)
1748	}
1749	if raw, _ := json.Marshal(MoveState{}); strings.Contains(string(raw), "recovery_inputs") {
1750	t.Fatalf("recovery_inputs is not omitempty: %s", raw)
1751	}
1752	if raw, _ := json.Marshal(MoveState{RecoveryInputs: &MoveInputs{Release: "v8"}}); !strings.Contains(string(raw), `"recovery_inputs":{`) {
1753	t.Fatalf("recovery_inputs missing: %s", raw)
1754	}
1755	}
1756
1757	// The legacy abort completion is the legacy record's only (Plan 5 Task 1):
1758	// it refuses a per-op move, which completes through MoveGuardedAbortComplete
1759	// once the target region has parked its lane. A legacy aborting record still
1760	// completes, unchanged.
1761	func TestLegacyAbortCompleteRefusesAPerOpMove(t *testing.T) {
1762	g := MoveGuard{ID: "mv_abcdefghijklmnopqrstuvwxyz", Incarnation: strings.Repeat("ab", 32)}
1763	rec := &Record{Move: &MoveState{ID: g.ID, Incarnation: g.Incarnation, State: MoveAborting}}
1764	if err := MoveAbortComplete()(rec); !errors.Is(err, ErrMoveConflict) || rec.Move == nil {
1765	t.Fatalf("legacy completion of a per-op abort: %v %+v", err, rec.Move)
1766	}
1767	if err := MoveGuardedAbortComplete(g, time.Now())(rec); err != nil || rec.Move != nil {
1768	t.Fatalf("guarded completion: %v %+v", err, rec.Move)
1769	}
1770	legacy := &Record{Move: &MoveState{To: "uk", State: MoveAborting}}
1771	if err := MoveAbortComplete()(legacy); err != nil || legacy.Move != nil {
1772	t.Fatalf("legacy completion of a legacy abort: %v %+v", err, legacy.Move)
1773	}
1774	}
1775