carlos / platform Public

Clone
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package hostclaim
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "reflect"
9 "strings"
10 "testing"
11 "time"
12
13 "github.com/carlosframework/platform/internal/release"
14)
15
16const (
17 moveA = "mv_abcdefghijklmnopqrstuvwxyz"
18 moveB = "mv_bcdefghijklmnopqrstuvwxyza"
19)
20
21// movingTestClaim is this file's fixture (delete_test.go already defines
22// servingClaim(t, ctx, s) with another shape). It is a VALID claim: the
23// holder's key, incarnation and location follow the grammar Validate
24// enforces, and the fixture asserts Validate() so it can never drift from
25// the real grammar.
26func movingTestClaim(t *testing.T) *Claim {
27 t.Helper()
28 created := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC)
29 inc, err := Incarnation("bab", "app", "a.go.example.com", created)
30 if err != nil {
31 t.Fatal(err)
32 }
33 c := &Claim{V: schemaVersion, Host: "a.go.example.com", Gen: 3,
34 Holder: &Holder{Family: "instance", Key: "apps/bab/app/instances/a.go.example.com.json", Account: "bab", App: "app", Incarnation: inc, CreatedAt: created, Location: "instances/a.go.example.com/" + inc, Stamp: 3},
35 Op: Op{Kind: KindServing, Exec: "ie-box"},
36 Touched: Touched{Complete: true, Regions: []string{"ie"}, Boxes: []string{"ie-box"}}}
37 if err := c.Validate(); err != nil {
38 t.Fatalf("fixture must validate: %v", err)
39 }
40 return c
41}
42
43func TestBeginMoveFromServingOnly(t *testing.T) {
44 now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
45 c := movingTestClaim(t)
46 if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(c); err != nil {
47 t.Fatal(err)
48 }
49 if c.Op.Kind != KindMoving || c.Op.ID != moveA || c.Op.Exec != "ie-box" || c.Op.By != "ops" || !c.Op.StartedAt.Equal(now) || len(c.Op.Participants) != 2 || c.Op.Participants[0] != "ie-box" || c.Op.Participants[1] != "uk-box" || c.Op.Intent["to"] != "uk" {
50 t.Fatalf("op after BeginMove: %+v", c.Op)
51 }
52 // A plain BeginMove carries the move's own three keys and nothing else:
53 // the admission and both enrollments are BeginAdmittedMove's.
54 if want := map[string]string{"to": "uk", "source": "ie-box", "target": "uk-box"}; !reflect.DeepEqual(c.Op.Intent, want) {
55 t.Fatalf("intent after BeginMove = %v, want %v", c.Op.Intent, want)
56 }
57 if c.Op.Phase != PhaseMoving {
58 t.Fatalf("phase after BeginMove: %q", c.Op.Phase)
59 }
60 if err := c.Validate(); err != nil {
61 t.Fatalf("claim after BeginMove must validate: %v", err)
62 }
63 // K12: nothing is touched yet; the source adds the region before its
64 // first staging round.
65 if len(c.Touched.Regions) != 1 || len(c.Touched.Boxes) != 1 {
66 t.Fatalf("touched grew at BeginMove: %+v", c.Touched)
67 }
68 // Not from serving.
69 if err := BeginMove(moveB, "ops", "ie-box", "uk-box", "uk", now)(c); !errors.Is(err, ErrWrongState) {
70 t.Fatalf("second move: %v", err)
71 }
72 // exec must be the source.
73 d := movingTestClaim(t)
74 if err := BeginMove(moveA, "ops", "us-box", "uk-box", "uk", now)(d); !errors.Is(err, ErrWrongState) {
75 t.Fatalf("source not exec: %v", err)
76 }
77 // A pending chore in the target region refuses (CC11).
78 e := movingTestClaim(t)
79 e.Obligations = []Obligation{{ID: "o1", Op: "x", Kind: "park-replica", Region: "uk", Retirement: "r", Owner: "region:uk"}}
80 if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(e); !errors.Is(err, ErrPendingChore) {
81 t.Fatalf("pending chore: %v", err)
82 }
83 // ... and in the target box.
84 f := movingTestClaim(t)
85 f.Obligations = []Obligation{{ID: "o1", Op: "x", Kind: "park-local", Box: "uk-box", Retirement: "r", Owner: "box:uk-box"}}
86 if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(f); !errors.Is(err, ErrPendingChore) {
87 t.Fatalf("pending box chore: %v", err)
88 }
89 // A claim with no holder is not movable.
90 g := movingTestClaim(t)
91 g.Holder = nil
92 if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(g); !errors.Is(err, ErrWrongState) {
93 t.Fatalf("no holder: %v", err)
94 }
95 // Missing inputs are malformed.
96 for _, args := range [][4]string{{"", "ie-box", "uk-box", "uk"}, {moveA, "", "uk-box", "uk"}, {moveA, "ie-box", "", "uk"}, {moveA, "ie-box", "uk-box", ""}} {
97 h := movingTestClaim(t)
98 if err := BeginMove(args[0], "ops", args[1], args[2], args[3], now)(h); !errors.Is(err, ErrMalformed) {
99 t.Fatalf("args %v: %v", args, err)
100 }
101 if h.Op.Kind != KindServing {
102 t.Fatalf("a refused BeginMove changed the op: %+v", h.Op)
103 }
104 }
105}
106
107func TestTouchAppendsOnceUnderTheOp(t *testing.T) {
108 c := movingTestClaim(t)
109 if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", time.Now())(c); err != nil {
110 t.Fatal(err)
111 }
112 if err := Touch(moveA, []string{"uk"}, nil)(c); err != nil {
113 t.Fatal(err)
114 }
115 if err := Touch(moveA, []string{"uk"}, []string{"uk-box"})(c); err != nil {
116 t.Fatal(err)
117 }
118 if len(c.Touched.Regions) != 2 || len(c.Touched.Boxes) != 2 || c.Touched.Boxes[1] != "uk-box" {
119 t.Fatalf("touched: %+v", c.Touched)
120 }
121 if err := Touch("mv_other0000000000000000000000", []string{"se"}, nil)(c); !errors.Is(err, ErrStaleOp) {
122 t.Fatalf("touch under another op: %v", err)
123 }
124 if len(c.Touched.Regions) != 2 {
125 t.Fatalf("a refused Touch changed touched: %+v", c.Touched)
126 }
127 if err := Touch("", []string{"se"}, nil)(c); !errors.Is(err, ErrStaleOp) {
128 t.Fatalf("touch under no op: %v", err)
129 }
130 if err := RequireMoveOp(moveA, KindMoving, "")(c); err != nil {
131 t.Fatal(err)
132 }
133 if err := RequireMoveOp(moveA, KindLanding, "")(c); !errors.Is(err, ErrWrongState) {
134 t.Fatalf("wrong kind: %v", err)
135 }
136 if err := RequireMoveOp(moveA, KindMoving, PhaseSealed)(c); !errors.Is(err, ErrWrongState) {
137 t.Fatalf("wrong phase: %v", err)
138 }
139 if err := RequireMoveOp(moveB, KindMoving, "")(c); !errors.Is(err, ErrStaleOp) {
140 t.Fatalf("wrong op: %v", err)
141 }
142 c.Op.Phase = PhaseSealed
143 if err := RequireMoveOp(moveA, KindMoving, PhaseSealed)(c); err != nil {
144 t.Fatalf("sealed: %v", err)
145 }
146}
147
148func TestReleaseMoveFromMovingNeedsNoChores(t *testing.T) {
149 now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
150 for _, phase := range []string{PhaseMoving} {
151 c := movingTestClaim(t)
152 if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(c); err != nil {
153 t.Fatal(err)
154 }
155 c.Op.Phase = phase
156 // A chore under this op blocks the release.
157 c.Obligations = []Obligation{{ID: "o1", Op: moveA, Kind: "park-replica", Region: "uk", Retirement: "r", Owner: "region:uk"}}
158 if err := ReleaseMove(moveA, "ie-box")(c); !errors.Is(err, ErrPendingChore) {
159 t.Fatalf("phase %q with a chore: %v", phase, err)
160 }
161 if c.Op.Kind != KindMoving {
162 t.Fatalf("a refused release changed the op: %+v", c.Op)
163 }
164 // A chore under another op does not.
165 c.Obligations[0].Op = "someone-else"
166 if err := ReleaseMove(moveB, "ie-box")(c); !errors.Is(err, ErrStaleOp) {
167 t.Fatalf("release under another op: %v", err)
168 }
169 if err := ReleaseMove(moveA, "ie-box")(c); err != nil {
170 t.Fatalf("phase %q: %v", phase, err)
171 }
172 if c.Op.Kind != KindServing || c.Op.ID != "" || c.Op.Exec != "ie-box" || c.Op.Journal["released_move"] != moveA || c.Op.Phase != "" || len(c.Op.Participants) != 0 {
173 t.Fatalf("op after release: %+v", c.Op)
174 }
175 if err := c.Validate(); err != nil {
176 t.Fatalf("claim after release must validate: %v", err)
177 }
178 }
179 // Not from an aborting move (ruling K7: that is ReleaseAbort's, after
180 // the lane park the abort queued).
181 a := movingTestClaim(t)
182 _ = BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(a)
183 a.Op.Phase = PhaseAborting
184 if err := ReleaseMove(moveA, "ie-box")(a); !errors.Is(err, ErrWrongState) {
185 t.Fatalf("release from aborting: %v", err)
186 }
187 // Not from serving, and not from a sealed move.
188 s := movingTestClaim(t)
189 if err := ReleaseMove(moveA, "ie-box")(s); !errors.Is(err, ErrStaleOp) {
190 t.Fatalf("release from serving: %v", err)
191 }
192 m := movingTestClaim(t)
193 _ = BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(m)
194 m.Op.Phase = PhaseSealed
195 if err := ReleaseMove(moveA, "ie-box")(m); !errors.Is(err, ErrWrongState) {
196 t.Fatalf("release from sealed: %v", err)
197 }
198 // Another kind under the same id is not a move.
199 l := movingTestClaim(t)
200 _ = BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(l)
201 l.Op.Kind = KindLanding
202 if err := ReleaseMove(moveA, "ie-box")(l); !errors.Is(err, ErrWrongState) {
203 t.Fatalf("release from landing: %v", err)
204 }
205 // The executor is named.
206 e := movingTestClaim(t)
207 _ = BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(e)
208 if err := ReleaseMove(moveA, "")(e); !errors.Is(err, ErrMalformed) {
209 t.Fatalf("release without exec: %v", err)
210 }
211}
212
213func TestReleaseMoveLeavesEvidence(t *testing.T) {
214 now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
215 plain := movingTestClaim(t)
216 if err := RequireReleasedMove(moveA)(plain); !errors.Is(err, ErrStaleOp) {
217 t.Fatalf("a plain serving claim satisfies the replay rule: %v", err)
218 }
219 c := movingTestClaim(t)
220 if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(c); err != nil {
221 t.Fatal(err)
222 }
223 if err := RequireReleasedMove(moveA)(c); !errors.Is(err, ErrStaleOp) {
224 t.Fatalf("a moving claim satisfies the replay rule: %v", err)
225 }
226 if err := ReleaseMove(moveA, "ie-box")(c); err != nil {
227 t.Fatal(err)
228 }
229 if err := RequireReleasedMove(moveA)(c); err != nil {
230 t.Fatalf("evidence after release: %v", err)
231 }
232 if err := RequireReleasedMove(moveB)(c); !errors.Is(err, ErrStaleOp) {
233 t.Fatalf("another move's id: %v", err)
234 }
235 if err := RequireReleasedMove("")(c); !errors.Is(err, ErrStaleOp) {
236 t.Fatalf("empty id: %v", err)
237 }
238 // A non-serving kind carrying the journal is not evidence.
239 c.Op.Kind, c.Op.ID = KindMoving, moveB
240 if err := RequireReleasedMove(moveA)(c); !errors.Is(err, ErrStaleOp) {
241 t.Fatalf("moving claim with a stale journal: %v", err)
242 }
243}
244
245// BeginMove and ReleaseMove own only the op: the holder (session, writer,
246// admissions, generation and their writes_lost), touched, inventory,
247// obligations of other ops and the rest are carried through unchanged.
248func TestMoveRoundTripPreservesTheRestOfTheClaim(t *testing.T) {
249 now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
250 c := movingTestClaim(t)
251 c.Holder.Generation = "gen-7"
252 c.Holder.Session = 9
253 c.Holder.WriterBox = "ie-box"
254 c.Holder.Admissions = []AdmissionRecord{{Region: "ie", Session: 9, Box: "ie-box", Generation: "gen-7", Mode: "admitted", Coverage: "covered", Covered: true,
255 WritesLost: &WritesLost{Box: "ie-box", Generation: "gen-6", Session: 8, RemoteMaxTXID: 41},
256 AdditionalWritesLost: []WritesLost{{Box: "ie-box", Generation: "gen-5", Session: 7, RemoteMaxTXID: 12}}}}
257 c.Obligations = []Obligation{{ID: "o1", Op: "old", Kind: "park-replica", Region: "se", Retirement: "r", Owner: "region:se"}}
258 c.Lost = []string{"se"}
259 c.RetiredIncarnations = []string{"abc"}
260 c.QuietAfter = map[string]time.Time{"se": now}
261 c.Inventory = map[string]InventoryEntry{"ie-box": {Layout: "nested", Paths: map[string]uint64{"/data/x": 3}}}
262 if err := c.Validate(); err != nil {
263 t.Fatalf("fixture: %v", err)
264 }
265 before := mustJSON(t, c)
266 if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(c); err != nil {
267 t.Fatal(err)
268 }
269 if err := ReleaseMove(moveA, "ie-box")(c); err != nil {
270 t.Fatal(err)
271 }
272 // Only the op differs.
273 var round Claim
274 if err := json.Unmarshal(before, &round); err != nil {
275 t.Fatal(err)
276 }
277 round.Op = c.Op
278 if !reflect.DeepEqual(round, *c) {
279 t.Fatalf("round trip changed more than the op:\nbefore %s\nafter %s", before, mustJSON(t, c))
280 }
281 if err := c.Validate(); err != nil {
282 t.Fatalf("after round trip: %v", err)
283 }
284}
285
286func mustJSON(t *testing.T, c *Claim) []byte {
287 t.Helper()
288 b, err := json.Marshal(c)
289 if err != nil {
290 t.Fatal(err)
291 }
292 return b
293}
294
295const stoppedSource = "moves/bab/app/h/mv/stopped-source.json"
296
297// movingToSealed walks a fresh moving claim to the state the flip needs:
298// source evidence journalled, exec none, phase sealed.
299func movingToSealed(t *testing.T) *Claim {
300 t.Helper()
301 c := movingTestClaim(t)
302 c.Holder.Generation = "gen-7" // the source's generation, which the flip must name
303 for i, tr := range []func(*Claim) error{
304 BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", time.Now()),
305 ExecToNone(moveA, "ie-box", stoppedSource, JournalEvidenceSource),
306 SetJournal(moveA, map[string]string{JournalSeal: `{"n":7}`, JournalCutSession: "5"}),
307 SetMovePhase(moveA, KindMoving, PhaseMoving, PhaseSealed),
308 } {
309 if err := tr(c); err != nil {
310 t.Fatalf("step %d: %v", i, err)
311 }
312 }
313 return c
314}
315
316func TestCutClaimCASesFollowK3AndK4(t *testing.T) {
317 c := movingTestClaim(t)
318 op := moveA
319 if err := BeginMove(op, "ops", "ie-box", "uk-box", "uk", time.Now())(c); err != nil {
320 t.Fatal(err)
321 }
322 // Only the executor gives execution up, and only citing evidence.
323 if err := ExecToNone(op, "uk-box", stoppedSource, JournalEvidenceSource)(c); !errors.Is(err, ErrWrongState) {
324 t.Fatalf("non-exec to none: %v", err)
325 }
326 if err := ExecToNone(op, "ie-box", "", JournalEvidenceSource)(c); !errors.Is(err, ErrMalformed) {
327 t.Fatalf("no evidence: %v", err)
328 }
329 if err := ExecToNone(op, "ie-box", stoppedSource, JournalSeal)(c); !errors.Is(err, ErrMalformed) {
330 t.Fatalf("evidence under a non-evidence key: %v", err)
331 }
332 if err := ExecToNone(moveB, "ie-box", stoppedSource, JournalEvidenceSource)(c); !errors.Is(err, ErrStaleOp) {
333 t.Fatalf("another op: %v", err)
334 }
335 // The flip needs the seal phase and exec none.
336 if err := Flip(op, "uk-box", 3, "")(c); !errors.Is(err, ErrWrongState) {
337 t.Fatalf("flip before sealed: %v", err)
338 }
339 if err := ExecToNone(op, "ie-box", stoppedSource, JournalEvidenceSource)(c); err != nil || c.Op.Exec != "" {
340 t.Fatalf("to none: %v %+v", err, c.Op)
341 }
342 // A crash replay of the same give-up is idempotent; other evidence is not.
343 if err := ExecToNone(op, "ie-box", stoppedSource, JournalEvidenceSource)(c); err != nil {
344 t.Fatalf("replay: %v", err)
345 }
346 if err := ExecToNone(op, "ie-box", "moves/other.json", JournalEvidenceSource)(c); !errors.Is(err, ErrWrongState) {
347 t.Fatalf("replay with other evidence: %v", err)
348 }
349 // With exec already none, an empty box is nobody: it must neither
350 // replay nor replace the evidence.
351 given := mustJSON(t, c)
352 for _, ev := range []string{stoppedSource, "moves/other.json"} {
353 if err := ExecToNone(op, "", ev, JournalEvidenceSource)(c); !errors.Is(err, ErrMalformed) {
354 t.Fatalf("empty box with %q: %v", ev, err)
355 }
356 if got := mustJSON(t, c); string(got) != string(given) {
357 t.Fatalf("empty box with %q mutated the claim: %s", ev, got)
358 }
359 }
360 // Exec none and evidence, but not sealed: still no flip.
361 if err := Flip(op, "uk-box", 3, "")(c); !errors.Is(err, ErrWrongState) {
362 t.Fatalf("flip before the seal phase: %v", err)
363 }
364 if err := SetJournal(op, map[string]string{JournalSeal: `{"n":7}`, JournalCutSession: "5"})(c); err != nil || c.Op.Journal[JournalSeal] != `{"n":7}` {
365 t.Fatalf("journal: %v", err)
366 }
367 if err := SetJournal(moveB, map[string]string{JournalSeal: "x"})(c); !errors.Is(err, ErrStaleOp) {
368 t.Fatalf("journal under another op: %v", err)
369 }
370 if err := SetMovePhase(op, KindLanding, PhaseMoving, PhaseSealed)(c); !errors.Is(err, ErrWrongState) {
371 t.Fatalf("phase under the wrong kind: %v", err)
372 }
373 if err := SetMovePhase(op, KindMoving, PhaseMoving, PhaseSealed)(c); err != nil || c.Op.Phase != PhaseSealed {
374 t.Fatalf("phase: %v", err)
375 }
376 if err := SetMovePhase(op, KindMoving, PhaseMoving, PhaseSealed)(c); !errors.Is(err, ErrWrongState) {
377 t.Fatal("phase replay")
378 }
379 // The flip's target must be the move's target.
380 if err := Flip(op, "us-box", 3, "")(c); !errors.Is(err, ErrWrongState) {
381 t.Fatalf("flip to a stranger: %v", err)
382 }
383 before := c.Holder.Stamp
384 c.Gen = 9 // what Update will have set
385 if err := Flip(op, "uk-box", before, "")(c); err != nil {
386 t.Fatal(err)
387 }
388 if c.Op.Kind != KindLanding || c.Op.Phase != "checking" || c.Op.Exec != "uk-box" || c.Op.Journal[JournalSourceStamp] != "3" || c.Op.Journal[JournalSrcGen] != "" || c.Holder.Stamp != 10 {
389 t.Fatalf("after flip: %+v stamp %d", c.Op, c.Holder.Stamp)
390 }
391 // holder.generation is untouched by the flip (section 7.5).
392 if c.Holder.Generation != "" {
393 t.Fatal("flip moved the generation")
394 }
395}
396
397// Flip is written by either participant (G12): the target may write it
398// after a source crash. A second write of the same flip is a no-op that
399// does not restamp the holder; a different flip is refused.
400func TestFlipIsIdempotentForEitherWriter(t *testing.T) {
401 c := movingToSealed(t)
402 c.Gen = 9
403 // The first flip must name the source's own stamp and generation: a
404 // wrong identity would be journalled as the rollback's return.
405 sealed := mustJSON(t, c)
406 for name, fn := range map[string]func(*Claim) error{
407 "first flip, wrong stamp": Flip(moveA, "uk-box", 4, "gen-7"),
408 "first flip, zero stamp": Flip(moveA, "uk-box", 0, "gen-7"),
409 "first flip, wrong generation": Flip(moveA, "uk-box", 3, "gen-8"),
410 "first flip, empty generation": Flip(moveA, "uk-box", 3, ""),
411 } {
412 if err := fn(c); !errors.Is(err, ErrWrongState) {
413 t.Fatalf("%s: %v", name, err)
414 }
415 if got := mustJSON(t, c); string(got) != string(sealed) {
416 t.Fatalf("%s mutated the claim:\nbefore %s\nafter %s", name, sealed, got)
417 }
418 }
419 if err := Flip(moveA, "uk-box", 3, "gen-7")(c); err != nil {
420 t.Fatal(err)
421 }
422 c.Gen = 10 // the replay observes the claim the flip produced
423 op := c.Op
424 if err := Flip(moveA, "uk-box", 3, "gen-7")(c); err != nil {
425 t.Fatalf("replay: %v", err)
426 }
427 if c.Holder.Stamp != 10 || !reflect.DeepEqual(c.Op, op) {
428 t.Fatalf("replay moved the claim: %+v stamp %d", c.Op, c.Holder.Stamp)
429 }
430 // The replay is checked against the journal, not the holder: the holder
431 // stamp is now F (10), not S (3).
432 flipped := mustJSON(t, c)
433 for name, fn := range map[string]func(*Claim) error{
434 "another stamp": Flip(moveA, "uk-box", 4, "gen-7"),
435 "the new holder stamp": Flip(moveA, "uk-box", 10, "gen-7"),
436 "another generation": Flip(moveA, "uk-box", 3, "gen-8"),
437 "an empty generation": Flip(moveA, "uk-box", 3, ""),
438 "another target": Flip(moveA, "us-box", 3, "gen-7"),
439 "another op": Flip(moveB, "uk-box", 3, "gen-7"),
440 } {
441 if err := fn(c); err == nil {
442 t.Fatalf("%s: a different flip was accepted", name)
443 }
444 if got := mustJSON(t, c); string(got) != string(flipped) {
445 t.Fatalf("%s mutated the claim", name)
446 }
447 }
448 // Once the landing has moved past checking, the replay is stale.
449 c.Op.Phase = "committing"
450 if err := Flip(moveA, "uk-box", 3, "gen-7")(c); !errors.Is(err, ErrWrongState) {
451 t.Fatalf("replay after the landing moved on: %v", err)
452 }
453}
454
455// The flip's holder.stamp is the gen the CAS produces: run it through the
456// real Update and compare.
457func TestFlipStampIsTheGenUpdateProduces(t *testing.T) {
458 ctx := context.Background()
459 s := release.NewDirStore(t.TempDir())
460 c := movingToSealed(t)
461 key, err := Key(c.Host)
462 if err != nil {
463 t.Fatal(err)
464 }
465 if _, err := s.Put(ctx, key, mustJSON(t, c), ""); err != nil {
466 t.Fatal(err)
467 }
468 o, err := Read(ctx, s, c.Host)
469 if err != nil {
470 t.Fatal(err)
471 }
472 o, err = Update(ctx, s, o, Flip(moveA, "uk-box", c.Holder.Stamp, "gen-7"))
473 if err != nil {
474 t.Fatal(err)
475 }
476 if o.Claim.Holder.Stamp != o.Claim.Gen || o.Claim.Gen != c.Gen+1 {
477 t.Fatalf("stamp %d gen %d, want both %d", o.Claim.Holder.Stamp, o.Claim.Gen, c.Gen+1)
478 }
479 if o.Claim.Op.Journal[JournalSourceStamp] != "3" || o.Claim.Op.Journal[JournalSrcGen] != "gen-7" {
480 t.Fatalf("journal: %v", o.Claim.Op.Journal)
481 }
482}
483
484func TestRevertNeedsEvidenceAndReturnsExecToSource(t *testing.T) {
485 c := movingTestClaim(t)
486 op := moveA
487 if err := BeginMove(op, "ops", "ie-box", "uk-box", "uk", time.Now())(c); err != nil {
488 t.Fatal(err)
489 }
490 scope := ParkScope{Location: "moves/" + c.Host + "/" + op, Incarnation: c.Holder.Incarnation, EvidenceAt: time.Now(), RequestLimit: 120 * time.Second}
491 ids := 0
492 newID := func() string { ids++; return fmt.Sprintf("id%d", ids) }
493 if err := Revert(op, "ie-box", "uk", &scope, newID)(c); !errors.Is(err, ErrWrongState) {
494 t.Fatalf("revert while exec held: %v", err)
495 }
496 if err := ExecToNone(op, "ie-box", stoppedSource, JournalEvidenceSource)(c); err != nil {
497 t.Fatal(err)
498 }
499 if err := Revert(op, "uk-box", "uk", &scope, newID)(c); !errors.Is(err, ErrWrongState) {
500 t.Fatalf("revert to the target: %v", err)
501 }
502 if err := Revert(op, "ie-box", "se", &scope, newID)(c); !errors.Is(err, ErrWrongState) {
503 t.Fatalf("revert naming a region the move never targeted: %v", err)
504 }
505 if err := Revert(op, "ie-box", "uk", nil, newID)(c); !errors.Is(err, ErrMalformed) {
506 t.Fatalf("revert with no staging scope: %v", err)
507 }
508 live := scope
509 live.Location = c.Holder.Location
510 if err := Revert(op, "ie-box", "uk", &live, newID)(c); !errors.Is(err, ErrMalformed) {
511 t.Fatalf("revert whose park would reach the live location: %v", err)
512 }
513 // The scope is this move's own lane and this holder's incarnation: not
514 // the host-wide staging prefix, not another op's lane, not another
515 // installation's incarnation.
516 hostWide := scope
517 hostWide.Location = "moves/" + c.Host
518 otherOp := scope
519 otherOp.Location = "moves/" + c.Host + "/" + moveB
520 otherInc := scope
521 otherInc.Incarnation = strings.Repeat("ab", 32)
522 preRevert := mustJSON(t, c)
523 for name, sc := range map[string]ParkScope{"host-wide": hostWide, "another op's lane": otherOp, "another incarnation": otherInc} {
524 sc := sc
525 if err := Revert(op, "ie-box", "uk", &sc, newID)(c); !errors.Is(err, ErrMalformed) {
526 t.Fatalf("revert with a %s scope: %v", name, err)
527 }
528 if got := mustJSON(t, c); string(got) != string(preRevert) {
529 t.Fatalf("a refused revert (%s scope) mutated the claim", name)
530 }
531 }
532 if ids != 0 {
533 t.Fatalf("a refused revert allocated %d ids", ids)
534 }
535 if c.Op.Kind != KindMoving || len(c.Obligations) != 0 {
536 t.Fatalf("a refused revert left changes: %+v", c.Op)
537 }
538 if err := Revert(op, "ie-box", "uk", &scope, newID)(c); err != nil || c.Op.Kind != KindReverting || c.Op.Exec != "ie-box" || c.Op.Journal[JournalRevPhase] != "evidence" {
539 t.Fatalf("revert: %v %+v", err, c.Op)
540 }
541 if len(c.Obligations) != 1 || c.Obligations[0].Kind != "park-replica" || c.Obligations[0].Region != "uk" || c.Obligations[0].Park == nil ||
542 c.Obligations[0].Owner != "region:uk" || c.Obligations[0].Op != op || c.Obligations[0].Park.Location != scope.Location {
543 t.Fatalf("revert queued no target staging park: %+v", c.Obligations)
544 }
545 c.Gen = 9
546 if err := c.Validate(); err != nil {
547 t.Fatalf("claim after revert must validate: %v", err)
548 }
549}
550
551// A revert keeps rev_phase if a replay already set it, and queues the
552// staging park once even when one is already owed under the op.
553func TestRevertKeepsRevPhaseAndQueuesTheParkOnce(t *testing.T) {
554 c := movingTestClaim(t)
555 if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", time.Now())(c); err != nil {
556 t.Fatal(err)
557 }
558 if err := ExecToNone(moveA, "ie-box", stoppedSource, JournalEvidenceSource)(c); err != nil {
559 t.Fatal(err)
560 }
561 scope := ParkScope{Location: "moves/" + c.Host + "/" + moveA, Incarnation: c.Holder.Incarnation, EvidenceAt: time.Now(), RequestLimit: time.Minute}
562 c.Obligations = []Obligation{{ID: "o1", Op: moveA, Kind: "park-replica", Region: "uk", Retirement: "r1", Owner: "region:uk", Park: &scope}}
563 if err := SetJournal(moveA, map[string]string{JournalRevPhase: "staging"})(c); err != nil {
564 t.Fatal(err)
565 }
566 if err := Revert(moveA, "ie-box", "uk", &scope, counter())(c); err != nil {
567 t.Fatal(err)
568 }
569 if c.Op.Journal[JournalRevPhase] != "staging" || len(c.Obligations) != 1 {
570 t.Fatalf("rev_phase %q, obligations %+v", c.Op.Journal[JournalRevPhase], c.Obligations)
571 }
572}
573
574// The cut's transitions own only the op, the journal, the revert's staging
575// park and (the flip) the holder's stamp; every other field is carried
576// through unchanged.
577func TestCutTransitionsPreserveTheRestOfTheClaim(t *testing.T) {
578 now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
579 fixture := func() *Claim {
580 c := movingTestClaim(t)
581 c.Holder.Generation = "gen-7"
582 c.Holder.Session = 9
583 c.Holder.WriterBox = "ie-box"
584 c.Holder.Admissions = []AdmissionRecord{{Region: "ie", Session: 9, Box: "ie-box", Generation: "gen-7", Mode: "admitted", Coverage: "covered", Covered: true,
585 WritesLost: &WritesLost{Box: "ie-box", Generation: "gen-6", Session: 8, RemoteMaxTXID: 41},
586 AdditionalWritesLost: []WritesLost{{Box: "ie-box", Generation: "gen-5", Session: 7, RemoteMaxTXID: 12}}}}
587 c.Obligations = []Obligation{{ID: "o1", Op: "old", Kind: "park-replica", Region: "se", Retirement: "r", Owner: "region:se"}}
588 c.Lost = []string{"se"}
589 c.RetiredIncarnations = []string{"abc"}
590 c.QuietAfter = map[string]time.Time{"se": now}
591 c.Inventory = map[string]InventoryEntry{"ie-box": {Layout: "nested", Paths: map[string]uint64{"/data/x": 3}}}
592 if err := c.Validate(); err != nil {
593 t.Fatalf("fixture: %v", err)
594 }
595 return c
596 }
597 scope := ParkScope{Location: "moves/a.go.example.com/" + moveA, Incarnation: fixture().Holder.Incarnation, EvidenceAt: now, RequestLimit: time.Minute}
598 begin := func() []func(*Claim) error {
599 return []func(*Claim) error{
600 BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now),
601 ExecToNone(moveA, "ie-box", stoppedSource, JournalEvidenceSource),
602 }
603 }
604 cases := map[string]struct {
605 steps []func(*Claim) error
606 restamp bool // the flip restamps the holder
607 parks int // staging parks the transition queues
608 }{
609 "flip": {restamp: true, steps: append(begin(),
610 SetJournal(moveA, map[string]string{JournalSeal: `{"n":7}`}),
611 SetMovePhase(moveA, KindMoving, PhaseMoving, PhaseSealed),
612 Flip(moveA, "uk-box", 3, "gen-7"))},
613 "revert": {parks: 1, steps: append(begin(), Revert(moveA, "ie-box", "uk", &scope, counter()))},
614 }
615 for name, tc := range cases {
616 t.Run(name, func(t *testing.T) {
617 c := fixture()
618 before := mustJSON(t, c)
619 for i, tr := range tc.steps {
620 if err := tr(c); err != nil {
621 t.Fatalf("step %d: %v", i, err)
622 }
623 }
624 if len(c.Obligations) != 1+tc.parks || !reflect.DeepEqual(c.Obligations[0], fixture().Obligations[0]) {
625 t.Fatalf("obligations of other ops changed: %+v", c.Obligations)
626 }
627 var round Claim
628 if err := json.Unmarshal(before, &round); err != nil {
629 t.Fatal(err)
630 }
631 round.Op, round.Obligations = c.Op, c.Obligations
632 if tc.restamp {
633 round.Holder.Stamp = c.Holder.Stamp
634 }
635 if !reflect.DeepEqual(round, *c) {
636 t.Fatalf("%s changed more than it owns:\nbefore %s\nafter %s", name, before, mustJSON(t, c))
637 }
638 c.Gen++ // what Update would do
639 if err := c.Validate(); err != nil {
640 t.Fatalf("after %s: %v", name, err)
641 }
642 })
643 }
644}
645
646// A revert replayed after a crash finds the claim already reverting with
647// the same op, region, scope and rev_phase: nil, no new ids, one park.
648func TestRevertReplayIsIdempotent(t *testing.T) {
649 c := movingTestClaim(t)
650 if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", time.Now())(c); err != nil {
651 t.Fatal(err)
652 }
653 if err := ExecToNone(moveA, "ie-box", stoppedSource, JournalEvidenceSource)(c); err != nil {
654 t.Fatal(err)
655 }
656 scope := ParkScope{Location: "moves/" + c.Host + "/" + moveA, Incarnation: c.Holder.Incarnation, EvidenceAt: time.Now().UTC(), RequestLimit: time.Minute}
657 ids := 0
658 newID := func() string { ids++; return fmt.Sprintf("id%d", ids) }
659 if err := Revert(moveA, "ie-box", "uk", &scope, newID)(c); err != nil {
660 t.Fatal(err)
661 }
662 after, usedIDs := mustJSON(t, c), ids
663 if err := Revert(moveA, "ie-box", "uk", &scope, newID)(c); err != nil {
664 t.Fatalf("replay: %v", err)
665 }
666 if got := mustJSON(t, c); string(got) != string(after) || ids != usedIDs || len(c.Obligations) != 1 {
667 t.Fatalf("replay changed the claim or allocated ids (%d, was %d): %s", ids, usedIDs, mustJSON(t, c))
668 }
669 // A replay that is not the same revert is refused, and changes nothing.
670 later := scope
671 later.EvidenceAt = scope.EvidenceAt.Add(time.Second)
672 for name, fn := range map[string]func(*Claim) error{
673 "another scope": Revert(moveA, "ie-box", "uk", &later, newID),
674 "another region": Revert(moveA, "ie-box", "se", &scope, newID),
675 "another source": Revert(moveA, "uk-box", "uk", &scope, newID),
676 "another op": Revert(moveB, "ie-box", "uk", &scope, newID),
677 } {
678 if err := fn(c); err == nil {
679 t.Fatalf("%s: accepted as a replay", name)
680 }
681 if got := mustJSON(t, c); string(got) != string(after) || ids != usedIDs {
682 t.Fatalf("%s changed the claim or allocated ids", name)
683 }
684 }
685 // Once the revert has moved past evidence, the replay is stale.
686 c.Op.Journal[JournalRevPhase] = "opened"
687 if err := Revert(moveA, "ie-box", "uk", &scope, newID)(c); !errors.Is(err, ErrWrongState) {
688 t.Fatalf("replay after the revert moved on: %v", err)
689 }
690}
691
692// parkFixture is a valid landing-shaped claim for the park queue helpers:
693// op moveA, a holder with an incarnation, and the scope helpers an op needs.
694func parkFixture(t *testing.T) (*Claim, func(location string) ParkScope) {
695 t.Helper()
696 c := movingTestClaim(t)
697 if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC))(c); err != nil {
698 t.Fatal(err)
699 }
700 scope := func(location string) ParkScope {
701 return ParkScope{Location: location, Incarnation: c.Holder.Incarnation, EvidenceAt: time.Date(2026, 10, 6, 9, 5, 0, 0, time.UTC), RequestLimit: 2 * time.Minute}
702 }
703 return c, scope
704}
705
706// inState puts a claim in the op state a park helper's lifecycle check reads:
707// the kind and phase, as the transitions that own them would leave it.
708func inState(c *Claim, kind, phase string) {
709 c.Op.Kind, c.Op.Phase = kind, phase
710}
711
712// The move's retirements ride the engine's existing kinds. A real store
713// round trip: Validate accepts what the helper wrote and it reads back.
714func TestLocationParkIsAParkReplicaTheStoreAccepts(t *testing.T) {
715 ctx := context.Background()
716 s := release.NewDirStore(t.TempDir())
717 o := servingClaim(t, ctx, s)
718 now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
719 o, err := Update(ctx, s, o, BeginMove(moveA, "ops", "adi-knitting", "uk-box", "uk", now))
720 if err != nil {
721 t.Fatal(err)
722 }
723 inc := o.Claim.Holder.Incarnation
724 scope := ParkScope{Location: "instances/" + testHost + "/" + inc, Incarnation: inc, EvidenceAt: now, RequestLimit: 2 * time.Minute}
725 // The commit's claim state: the source's location is parked from here.
726 if o, err = Update(ctx, s, o, func(c *Claim) error { inState(c, KindLanding, "committed"); return nil }); err != nil {
727 t.Fatal(err)
728 }
729 o, err = Update(ctx, s, o, QueueLocationPark(moveA, "ie", "adi-knitting", scope, counter()))
730 if err != nil {
731 t.Fatal(err)
732 }
733 o, err = Read(ctx, s, testHost)
734 if err != nil {
735 t.Fatal(err)
736 }
737 if err := o.Claim.Validate(); err != nil {
738 t.Fatal(err)
739 }
740 if len(o.Claim.Obligations) != 2 {
741 t.Fatalf("obligations: %+v", o.Claim.Obligations)
742 }
743 r, l := o.Claim.Obligations[0], o.Claim.Obligations[1]
744 if r.Kind != "park-replica" || r.Op != moveA || r.Region != "ie" || r.Owner != "region:ie" || r.Park == nil || !sameParkScope(*r.Park, scope) || r.Retirement == "" || r.ID == "" || r.ID == r.Retirement {
745 t.Fatalf("replica park: %+v", r)
746 }
747 if l.Kind != "park-local" || l.Op != moveA || l.Box != "adi-knitting" || l.Owner != "box:adi-knitting" || l.Park != nil || l.Region != "" || l.Retirement == "" {
748 t.Fatalf("local park: %+v", l)
749 }
750 // The same call again, across the store: nothing new, no ids spent.
751 ids := 0
752 again := func() string { ids++; return fmt.Sprintf("late_%d", ids) }
753 if o, err = Update(ctx, s, o, QueueLocationPark(moveA, "ie", "adi-knitting", scope, again)); err != nil || ids != 0 || len(o.Claim.Obligations) != 2 {
754 t.Fatalf("replay: ids=%d obligations=%d err=%v", ids, len(o.Claim.Obligations), err)
755 }
756 // Completing both by the existing chore completion ends them.
757 for _, ob := range o.Claim.Obligations {
758 if o, err = Update(ctx, s, o, CompleteObligation(ob.Op, ob.ID)); err != nil {
759 t.Fatal(err)
760 }
761 }
762 if len(o.Claim.Obligations) != 0 {
763 t.Fatal("chores left")
764 }
765}
766
767func TestStagingParkScopeIsTheLane(t *testing.T) {
768 c, scope := parkFixture(t)
769 lane := scope("moves/" + c.Host + "/" + moveA)
770 if err := QueueStagingPark(moveA, "uk", lane, counter())(c); err != nil {
771 t.Fatal(err)
772 }
773 if len(c.Obligations) != 1 {
774 t.Fatalf("obligations: %+v", c.Obligations)
775 }
776 o := c.Obligations[0]
777 if o.Kind != "park-replica" || o.Region != "uk" || o.Owner != "region:uk" || o.Op != moveA || o.Park.Location != "moves/"+c.Host+"/"+moveA {
778 t.Fatalf("staging park: %+v", o)
779 }
780 c.Gen++
781 if err := c.Validate(); err != nil {
782 t.Fatal(err)
783 }
784 // Anything but this move's own lane is refused, and nothing is queued.
785 for name, location := range map[string]string{
786 "the live location": "instances/" + c.Host + "/" + c.Holder.Incarnation,
787 "the legacy prefix": "instances/" + c.Host,
788 "the staging prefix": "moves/" + c.Host,
789 "another move": "moves/" + c.Host + "/" + moveB,
790 "another host": "moves/other.example.com/" + moveA,
791 "below the lane": "moves/" + c.Host + "/" + moveA + "/x",
792 } {
793 d, _ := parkFixture(t)
794 bad := scope(location)
795 ids := 0
796 if err := QueueStagingPark(moveA, "uk", bad, func() string { ids++; return "x" })(d); err == nil || len(d.Obligations) != 0 || ids != 0 {
797 t.Errorf("%s: accepted (%v), obligations %d", name, err, len(d.Obligations))
798 }
799 }
800 // Another holder's incarnation is refused.
801 d, _ := parkFixture(t)
802 foreign := scope("moves/" + d.Host + "/" + moveA)
803 foreign.Incarnation = strings.Repeat("a", 64)
804 if err := QueueStagingPark(moveA, "uk", foreign, counter())(d); !errors.Is(err, ErrMalformed) {
805 t.Errorf("foreign incarnation: %v", err)
806 }
807 // A stale op is refused.
808 if err := QueueStagingPark(moveB, "uk", lane, counter())(d); !errors.Is(err, ErrStaleOp) {
809 t.Errorf("stale op: %v", err)
810 }
811}
812
813// A revert already queued the lane's park: queueing it again is a no-op,
814// and a different scope for the same lane is refused, not overwritten.
815func TestStagingParkAfterRevertIsTheSameChore(t *testing.T) {
816 c, scope := parkFixture(t)
817 if err := ExecToNone(moveA, "ie-box", stoppedSource, JournalEvidenceSource)(c); err != nil {
818 t.Fatal(err)
819 }
820 lane := scope("moves/" + c.Host + "/" + moveA)
821 if err := Revert(moveA, "ie-box", "uk", &lane, counter())(c); err != nil {
822 t.Fatal(err)
823 }
824 ids := 0
825 newID := func() string { ids++; return "z" }
826 if err := QueueStagingPark(moveA, "uk", lane, newID)(c); err != nil || ids != 0 || len(c.Obligations) != 1 {
827 t.Fatalf("same lane: %v ids=%d obligations=%d", err, ids, len(c.Obligations))
828 }
829 later := lane
830 later.EvidenceAt = lane.EvidenceAt.Add(time.Second)
831 if err := QueueStagingPark(moveA, "uk", later, newID)(c); !errors.Is(err, ErrWrongState) || ids != 0 || len(c.Obligations) != 1 {
832 t.Fatalf("changed scope: %v ids=%d", err, ids)
833 }
834}
835
836func TestLocationParkAndStagingParkCoexistInOneRegion(t *testing.T) {
837 // A rolled-back target at cleaning owes the lane and the location in the
838 // same region under one op.
839 c, scope := parkFixture(t)
840 inState(c, KindRestoring, "released")
841 lane := scope("moves/" + c.Host + "/" + moveA)
842 loc := scope(c.Holder.Location)
843 n := counter()
844 for i := 0; i < 2; i++ {
845 if err := QueueStagingPark(moveA, "uk", lane, n)(c); err != nil {
846 t.Fatal(err)
847 }
848 if err := QueueLocationPark(moveA, "uk", "", loc, n)(c); err != nil {
849 t.Fatal(err)
850 }
851 }
852 if len(c.Obligations) != 2 {
853 t.Fatalf("obligations: %+v", c.Obligations)
854 }
855 c.Gen++
856 if err := c.Validate(); err != nil {
857 t.Fatal(err)
858 }
859 // The location's local chore is per box, not per region.
860 if err := QueueLocationPark(moveA, "uk", "uk-box", loc, n)(c); err != nil || len(c.Obligations) != 3 {
861 t.Fatalf("add the box: %v %+v", err, c.Obligations)
862 }
863 if err := QueueLocationPark(moveA, "uk", "uk-box", loc, n)(c); err != nil || len(c.Obligations) != 3 {
864 t.Fatalf("box twice: %v %+v", err, c.Obligations)
865 }
866}
867
868// A whole location is retired only where the region has stopped being
869// authoritative: the source after the commit (landing/committed), and a
870// rolled-back target (restoring, from its release). Every other state, and
871// the other region in either state, is refused with nothing queued and no
872// ids spent.
873func TestLocationParkOnlyWhereTheRegionIsRetired(t *testing.T) {
874 type tc struct {
875 kind, phase, region, box string
876 allowed bool
877 }
878 cases := map[string]tc{
879 "source after the commit": {KindLanding, "committed", "ie", "ie-box", true},
880 "source after the commit, region only": {KindLanding, "committed", "ie", "", true},
881 "rolled-back target at cleaning": {KindRestoring, "released", "uk", "uk-box", true},
882 "rolled-back target after the source": {KindRestoring, "claimed", "uk", "uk-box", true},
883 "source before the cut": {KindMoving, PhaseMoving, "ie", "ie-box", false},
884 "source sealed": {KindMoving, PhaseSealed, "ie", "ie-box", false},
885 "source in the abort": {KindMoving, PhaseAborting, "ie", "ie-box", false},
886 "source while the landing checks": {KindLanding, "checking", "ie", "ie-box", false},
887 "source while the landing probes": {KindLanding, "probing", "ie", "ie-box", false},
888 "source while reverting": {KindReverting, "evidence", "ie", "ie-box", false},
889 "source after the rollback began": {KindRestoring, "released", "ie", "ie-box", false},
890 "target after the commit": {KindLanding, "committed", "uk", "uk-box", false},
891 "target before the commit": {KindLanding, "probing", "uk", "uk-box", false},
892 "target before its release": {KindRestoring, "releasing", "uk", "uk-box", false},
893 "target still serving": {KindServing, "", "uk", "uk-box", false},
894 "the target's box from the source": {KindLanding, "committed", "ie", "uk-box", false},
895 "the source's box from a rolled-back uk": {KindRestoring, "released", "uk", "ie-box", false},
896 }
897 for name, tc := range cases {
898 t.Run(name, func(t *testing.T) {
899 c, scope := parkFixture(t)
900 inState(c, tc.kind, tc.phase)
901 ids := 0
902 newID := func() string { ids++; return fmt.Sprintf("p%d", ids) }
903 before := mustJSON(t, c)
904 err := QueueLocationPark(moveA, tc.region, tc.box, scope(c.Holder.Location), newID)(c)
905 if tc.allowed {
906 if err != nil || len(c.Obligations) == 0 {
907 t.Fatalf("refused: %v", err)
908 }
909 c.Gen++
910 if err := c.Validate(); err != nil {
911 t.Fatal(err)
912 }
913 return
914 }
915 if !errors.Is(err, ErrWrongState) || ids != 0 || string(mustJSON(t, c)) != string(before) {
916 t.Fatalf("err=%v ids=%d, claim changed=%v", err, ids, string(mustJSON(t, c)) != string(before))
917 }
918 })
919 }
920}
921
922// Location and residue parks name holder.location, whichever form the
923// holder uses (spec §6.4): the legacy instances/<host> or the
924// incarnation-scoped instances/<host>/<inc>, never the other one. A trailing
925// slash is normalised away on both sides.
926func TestParkLocationMustBeTheHoldersLocation(t *testing.T) {
927 for _, form := range []string{"scoped", "legacy"} {
928 t.Run(form, func(t *testing.T) {
929 c, scope := parkFixture(t)
930 scoped := "instances/" + c.Host + "/" + c.Holder.Incarnation
931 legacy := "instances/" + c.Host
932 own, other := scoped, legacy
933 if form == "legacy" {
934 own, other = legacy, scoped
935 }
936 c.Holder.Location = own
937 inState(c, KindLanding, "committed")
938 if err := c.Validate(); err != nil {
939 t.Fatal(err)
940 }
941 ids := 0
942 newID := func() string { ids++; return fmt.Sprintf("p%d", ids) }
943 before := mustJSON(t, c)
944 for name, fn := range map[string]func(*Claim) error{
945 "location": QueueLocationPark(moveA, "ie", "ie-box", scope(other), newID),
946 "residue": QueueResidueParks(moveA, "ie", scope(other), []string{"gen-1"}, newID),
947 "slash/other": QueueLocationPark(moveA, "ie", "", scope(other+"/"), newID),
948 } {
949 if err := fn(c); !errors.Is(err, ErrMalformed) && !errors.Is(err, ErrWrongState) || ids != 0 || string(mustJSON(t, c)) != string(before) {
950 t.Errorf("%s: err=%v ids=%d", name, err, ids)
951 }
952 }
953 // The holder's own form is accepted, with or without a trailing slash,
954 // and what is stored has none.
955 if err := QueueLocationPark(moveA, "ie", "", scope(own+"/"), newID)(c); err != nil {
956 t.Fatalf("own location with a slash: %v", err)
957 }
958 if err := QueueResidueParks(moveA, "ie", scope(own), []string{"gen-1"}, newID)(c); err != nil {
959 t.Fatalf("own location residue: %v", err)
960 }
961 if len(c.Obligations) != 2 || c.Obligations[0].Park.Location != own || c.Obligations[1].Park.Location != own {
962 t.Fatalf("stored: %+v", c.Obligations)
963 }
964 })
965 }
966}
967
968func TestLocationParkRefusals(t *testing.T) {
969 c, scope := parkFixture(t)
970 inState(c, KindLanding, "committed")
971 loc := scope(c.Holder.Location)
972 lane := scope("moves/" + c.Host + "/" + moveA)
973 bad := loc
974 bad.RequestLimit = 0
975 changed := loc
976 changed.EvidenceAt = loc.EvidenceAt.Add(time.Hour)
977 if err := QueueLocationPark(moveA, "ie", "", loc, counter())(c); err != nil {
978 t.Fatal(err)
979 }
980 for name, fn := range map[string]func(*Claim) error{
981 "a lane through the location door": QueueLocationPark(moveA, "ie", "", lane, counter()),
982 "no region": QueueLocationPark(moveA, "", "", loc, counter()),
983 "an invalid scope": QueueLocationPark(moveA, "ie", "", bad, counter()),
984 "another scope, same chore": QueueLocationPark(moveA, "ie", "", changed, counter()),
985 "another op": QueueLocationPark(moveB, "ie", "", loc, counter()),
986 "no id source": QueueLocationPark(moveA, "ie", "", loc, nil),
987 } {
988 before := mustJSON(t, c)
989 if err := fn(c); err == nil {
990 t.Errorf("%s: accepted", name)
991 }
992 if string(mustJSON(t, c)) != string(before) {
993 t.Errorf("%s: changed the claim", name)
994 }
995 }
996}
997
998func TestResidueParksArePerGeneration(t *testing.T) {
999 c, scope := parkFixture(t)
1000 loc := scope("instances/" + c.Host + "/" + c.Holder.Incarnation)
1001 n := counter()
1002 gens := []string{"gen-4", RootGeneration, "gen-9"}
1003 if err := QueueResidueParks(moveA, "uk", loc, gens, n)(c); err != nil {
1004 t.Fatal(err)
1005 }
1006 if len(c.Obligations) != 3 {
1007 t.Fatalf("obligations: %+v", c.Obligations)
1008 }
1009 seen := map[string]bool{}
1010 for _, o := range c.Obligations {
1011 if o.Kind != "park-generation" || o.Op != moveA || o.Region != "uk" || o.Owner != "region:uk" || o.Park == nil || !sameParkScope(*o.Park, loc) || o.Retirement == "" || o.Box != "" {
1012 t.Fatalf("residue park: %+v", o)
1013 }
1014 seen[o.Generation] = true
1015 }
1016 if !seen["gen-4"] || !seen["gen-9"] || !seen["@root"] {
1017 t.Fatalf("generations: %v", seen)
1018 }
1019 c.Gen++
1020 if err := c.Validate(); err != nil {
1021 t.Fatal(err)
1022 }
1023 // Each chore owns its scope: editing one does not touch another.
1024 c.Obligations[0].Park.RequestLimit = time.Hour
1025 if c.Obligations[1].Park.RequestLimit == time.Hour {
1026 t.Fatal("chores share one scope")
1027 }
1028 // A double call, a repeated generation in the list and a superset add
1029 // only what is missing, and spend ids only for that.
1030 ids := 0
1031 next := func() string { ids++; return fmt.Sprintf("n%d", ids) }
1032 loc.RequestLimit = 2 * time.Minute
1033 c.Obligations[0].Park.RequestLimit = 2 * time.Minute
1034 if err := QueueResidueParks(moveA, "uk", loc, append(gens, "gen-4", "gen-11", "gen-11"), next)(c); err != nil {
1035 t.Fatal(err)
1036 }
1037 if len(c.Obligations) != 4 || ids != 2 || c.Obligations[3].Generation != "gen-11" {
1038 t.Fatalf("superset: %d obligations, %d ids", len(c.Obligations), ids)
1039 }
1040 // No generations is nothing to do; an empty name is malformed.
1041 if err := QueueResidueParks(moveA, "uk", loc, nil, next)(c); err != nil || len(c.Obligations) != 4 {
1042 t.Fatalf("none: %v", err)
1043 }
1044 if err := QueueResidueParks(moveA, "uk", loc, []string{"gen-12", ""}, next)(c); !errors.Is(err, ErrMalformed) {
1045 t.Fatalf("empty generation: %v", err)
1046 }
1047 // The staging lane is not a residue location.
1048 if err := QueueResidueParks(moveA, "uk", scope("moves/"+c.Host+"/"+moveA), []string{"gen-1"}, next)(c); err == nil {
1049 t.Fatal("residue parked a lane")
1050 }
1051}
1052
1053func TestLocalParkAndTheEarlierStampsPaths(t *testing.T) {
1054 c, _ := parkFixture(t)
1055 c.Inventory = map[string]InventoryEntry{"uk-box": {Layout: "nested", Paths: map[string]uint64{"/data/a": 4, "/data/b": 7}}}
1056 if !InventoryHasStamp(c, "uk-box", 4) || InventoryHasStamp(c, "uk-box", 5) || InventoryHasStamp(c, "ie-box", 4) || InventoryHasStamp(c, "uk-box", 0) {
1057 t.Fatal("InventoryHasStamp")
1058 }
1059 ids := 0
1060 newID := func() string { ids++; return fmt.Sprintf("l%d", ids) }
1061 if err := QueueLocalPark(moveA, "uk-box", newID)(c); err != nil {
1062 t.Fatal(err)
1063 }
1064 if err := QueueLocalPark(moveA, "uk-box", newID)(c); err != nil || ids != 2 || len(c.Obligations) != 1 {
1065 t.Fatalf("double call: %v ids=%d obligations=%d", err, ids, len(c.Obligations))
1066 }
1067 o := c.Obligations[0]
1068 if o.Kind != "park-local" || o.Box != "uk-box" || o.Owner != "box:uk-box" || o.Op != moveA || o.Region != "" || o.Park != nil {
1069 t.Fatalf("local park: %+v", o)
1070 }
1071 c.Gen++
1072 if err := c.Validate(); err != nil {
1073 t.Fatal(err)
1074 }
1075 // The replay with S' paths still there must not complete the chore.
1076 if err := CompleteLocalPark(moveA, "uk-box", 4)(c); !errors.Is(err, ErrWrongState) || len(c.Obligations) != 1 {
1077 t.Fatalf("complete with paths left: %v", err)
1078 }
1079 // With no S' paths the chore is completed under the op; a second time is a no-op.
1080 c.Inventory["uk-box"] = InventoryEntry{Layout: "nested", Paths: map[string]uint64{"/data/b": 7}}
1081 if err := CompleteLocalPark(moveA, "uk-box", 4)(c); err != nil || len(c.Obligations) != 0 {
1082 t.Fatalf("complete: %v %+v", err, c.Obligations)
1083 }
1084 if err := CompleteLocalPark(moveA, "uk-box", 4)(c); err != nil {
1085 t.Fatalf("replay: %v", err)
1086 }
1087 // It is the op's own chore only.
1088 if err := QueueLocalPark(moveA, "uk-box", newID)(c); err != nil {
1089 t.Fatal(err)
1090 }
1091 if err := CompleteLocalPark(moveB, "uk-box", 4)(c); !errors.Is(err, ErrStaleOp) || len(c.Obligations) != 1 {
1092 t.Fatalf("another op: %v", err)
1093 }
1094 if err := QueueLocalPark(moveA, "", newID)(c); err == nil {
1095 t.Fatal("a local park with no box")
1096 }
1097}
1098
1099// The park queue helpers own only the obligations they append. Every
1100// other claim field, and every other op's chore, is carried through.
1101func TestParkQueueHelpersPreserveTheRestOfTheClaim(t *testing.T) {
1102 now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)
1103 build := func() *Claim {
1104 c := movingTestClaim(t)
1105 c.Holder.Generation = "gen-7"
1106 c.Holder.Session = 9
1107 c.Holder.WriterBox = "ie-box"
1108 c.Holder.Admissions = []AdmissionRecord{{Region: "ie", Session: 9, Box: "ie-box", Generation: "gen-7", Mode: "admitted", Coverage: "covered", Covered: true,
1109 WritesLost: &WritesLost{Box: "ie-box", Generation: "gen-6", Session: 8, RemoteMaxTXID: 41},
1110 AdditionalWritesLost: []WritesLost{{Box: "ie-box", Generation: "gen-5", Session: 7, RemoteMaxTXID: 12}}}}
1111 c.Obligations = []Obligation{{ID: "o1", Op: "old", Kind: "park-replica", Region: "se", Retirement: "r", Owner: "region:se"}}
1112 c.Lost = []string{"se"}
1113 c.RetiredIncarnations = []string{"abc"}
1114 c.QuietAfter = map[string]time.Time{"se": now}
1115 c.Touched = Touched{Complete: true, Regions: []string{"ie", "uk"}, Boxes: []string{"ie-box", "uk-box"}}
1116 c.Inventory = map[string]InventoryEntry{"ie-box": {Layout: "nested", Paths: map[string]uint64{"/data/x": 3}}}
1117 if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(c); err != nil {
1118 t.Fatal(err)
1119 }
1120 inState(c, KindLanding, "committed")
1121 if err := c.Validate(); err != nil {
1122 t.Fatalf("fixture: %v", err)
1123 }
1124 return c
1125 }
1126 base := build()
1127 scope := func(loc string) ParkScope {
1128 return ParkScope{Location: loc, Incarnation: base.Holder.Incarnation, EvidenceAt: now, RequestLimit: time.Minute}
1129 }
1130 loc := scope(base.Holder.Location)
1131 lane := scope("moves/" + base.Host + "/" + moveA)
1132 cases := map[string]struct {
1133 fn func(*Claim) error
1134 want int
1135 }{
1136 "location+box": {QueueLocationPark(moveA, "ie", "ie-box", loc, counter()), 2},
1137 "staging": {QueueStagingPark(moveA, "uk", lane, counter()), 1},
1138 "local": {QueueLocalPark(moveA, "uk-box", counter()), 1},
1139 "residue": {QueueResidueParks(moveA, "uk", loc, []string{"gen-1", RootGeneration}, counter()), 2},
1140 }
1141 for name, tc := range cases {
1142 t.Run(name, func(t *testing.T) {
1143 c := build()
1144 before := mustJSON(t, c)
1145 if err := tc.fn(c); err != nil {
1146 t.Fatal(err)
1147 }
1148 if len(c.Obligations) != 1+tc.want || !reflect.DeepEqual(c.Obligations[0], build().Obligations[0]) {
1149 t.Fatalf("obligations: %+v", c.Obligations)
1150 }
1151 var round Claim
1152 if err := json.Unmarshal(before, &round); err != nil {
1153 t.Fatal(err)
1154 }
1155 round.Obligations = c.Obligations
1156 if !reflect.DeepEqual(round, *c) {
1157 t.Fatalf("changed more than the obligations:\nbefore %s\nafter %s", before, mustJSON(t, c))
1158 }
1159 c.Gen++
1160 if err := c.Validate(); err != nil {
1161 t.Fatal(err)
1162 }
1163 })
1164 }
1165}
1166
1167// probedLanding drives the claim to landing/probed the way the landing does:
1168// the flip, then every record phase mirrored through SetMovePhase. The
1169// target's probe admission is on the holder, its predecessor in region ie.
1170func probedLanding(t *testing.T) *Claim {
1171 t.Helper()
1172 c := movingToSealed(t)
1173 c.Gen = 7
1174 if err := Flip(moveA, "uk-box", 3, "gen-7")(c); err != nil {
1175 t.Fatal(err)
1176 }
1177 for _, step := range [][2]string{{"checking", "parking"}, {"parking", "building"}, {"building", "published"}, {"published", "local"}, {"local", "configured"}, {"configured", "restored"}, {"restored", "probing"}, {"probing", "probed"}} {
1178 if err := SetMovePhase(moveA, KindLanding, step[0], step[1])(c); err != nil {
1179 t.Fatal(err)
1180 }
1181 }
1182 c.Holder.Session = 9
1183 c.Holder.Admissions = []AdmissionRecord{{Region: "uk", PredecessorRegion: "ie", Session: 9, Box: "uk-box", Generation: moveA, Predecessor: "gen-7", Mode: "cold", Coverage: "sealed"}}
1184 return c
1185}
1186
1187func commitScopes(c *Claim) (src, stg ParkScope) {
1188 at := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC)
1189 src = ParkScope{Location: c.Holder.Location, Incarnation: c.Holder.Incarnation, EvidenceAt: at, RequestLimit: 120 * time.Second}
1190 stg = ParkScope{Location: "moves/" + c.Host + "/" + moveA, Incarnation: c.Holder.Incarnation, EvidenceAt: at, RequestLimit: 120 * time.Second}
1191 return src, stg
1192}
1193
1194func obligationKinds(c *Claim) map[string]int {
1195 kinds := map[string]int{}
1196 for _, o := range c.Obligations {
1197 kinds[o.Kind+"/"+o.Region+o.Box]++
1198 }
1199 return kinds
1200}
1201
1202// wCommitCoverage, wCommitReplayed, H3, H5(b).
1203func TestCommitQueuesObligationsAndCoversTheSourceRegion(t *testing.T) {
1204 for _, from := range []string{"probed"} {
1205 c := probedLanding(t)
1206 c.Op.Phase = from
1207 // The source box holds paths of the stay the move leaves: stamp S = 3.
1208 c.Inventory = map[string]InventoryEntry{"ie-box": {Layout: "flat", Paths: map[string]uint64{"/data/a": 3}}}
1209 stamp, session, gen := c.Holder.Stamp, c.Holder.Session, c.Holder.Generation
1210 ids := 0
1211 newID := func() string { ids++; return fmt.Sprintf("id%d", ids) }
1212 src, stg := commitScopes(c)
1213 if err := Commit(moveA, "ie", "ie-box", "uk", src, stg, newID)(c); err != nil {
1214 t.Fatalf("from %s: %v", from, err)
1215 }
1216 if c.Op.Kind != KindLanding || c.Op.Phase != PhaseCommitted || c.Op.Exec != "uk-box" {
1217 t.Fatalf("after commit: %+v", c.Op)
1218 }
1219 // H3: the holder's stamp, session and generation are the flip's, untouched.
1220 if c.Holder.Stamp != stamp || c.Holder.Session != session || c.Holder.Generation != gen {
1221 t.Fatalf("commit touched the holder: %+v", c.Holder)
1222 }
1223 k := obligationKinds(c)
1224 if k["park-replica/ie"] != 1 || k["park-local/ie-box"] != 1 || k["park-replica/uk"] != 1 || len(c.Obligations) != 3 {
1225 t.Fatalf("obligations: %+v", c.Obligations)
1226 }
1227 for _, o := range c.Obligations {
1228 if o.Kind == "park-replica" && o.Region == "ie" && (o.Park == nil || o.Park.Location != c.Holder.Location || o.Owner != "region:ie") {
1229 t.Fatalf("source park: %+v", o)
1230 }
1231 if o.Kind == "park-replica" && o.Region == "uk" && (o.Park == nil || o.Park.Location != stg.Location) {
1232 t.Fatalf("staging park: %+v", o)
1233 }
1234 }
1235 if !c.Holder.Admissions[0].Covered {
1236 t.Fatal("the source-region predecessor is not covered (wCommitCoverage)")
1237 }
1238 if err := c.Validate(); err != nil {
1239 t.Fatalf("the committed claim must validate: %v", err)
1240 }
1241 // Replay: nothing doubles and no ids are minted (wCommitReplayed).
1242 before, usedIDs := mustJSON(t, c), ids
1243 if err := Commit(moveA, "ie", "ie-box", "uk", src, stg, newID)(c); err != nil || ids != usedIDs || string(mustJSON(t, c)) != string(before) {
1244 t.Fatalf("replay: %v ids %d->%d", err, usedIDs, ids)
1245 }
1246 }
1247}
1248
1249// H5(b): the source's park-local is queued only when the source box lists
1250// paths of the journalled source stamp S.
1251func TestCommitQueuesTheSourceParkLocalOnlyForTheSourceStamp(t *testing.T) {
1252 for name, inv := range map[string]map[string]InventoryEntry{
1253 "no inventory": nil,
1254 "other stamp only": {"ie-box": {Layout: "flat", Paths: map[string]uint64{"/data/a": 2}}},
1255 "stamp on another box": {"uk-box": {Layout: "flat", Paths: map[string]uint64{"/data/a": 3}}},
1256 } {
1257 c := probedLanding(t)
1258 c.Inventory = inv
1259 ids := 0
1260 newID := func() string { ids++; return fmt.Sprintf("id%d", ids) }
1261 src, stg := commitScopes(c)
1262 if err := Commit(moveA, "ie", "ie-box", "uk", src, stg, newID)(c); err != nil {
1263 t.Fatalf("%s: %v", name, err)
1264 }
1265 k := obligationKinds(c)
1266 if k["park-replica/ie"] != 1 || k["park-replica/uk"] != 1 || len(c.Obligations) != 2 {
1267 t.Fatalf("%s: obligations %+v", name, c.Obligations)
1268 }
1269 if !c.Holder.Admissions[0].Covered {
1270 t.Fatalf("%s: the source region is not covered", name)
1271 }
1272 }
1273}
1274
1275// wCommitInterrupted: a commit that is refused leaves the claim as it was.
1276func TestCommitRefusesAnythingButAProbingTargetLanding(t *testing.T) {
1277 newID := func() string { return "x" }
1278 type commitFn = func(c *Claim, src, stg ParkScope) func(*Claim) error
1279 std := func(op, region, box string) commitFn {
1280 return func(c *Claim, src, stg ParkScope) func(*Claim) error {
1281 return Commit(op, region, box, "uk", src, stg, newID)
1282 }
1283 }
1284 for name, tc := range map[string]struct {
1285 mutate func(c *Claim, src, stg *ParkScope)
1286 fn commitFn
1287 want error
1288 }{
1289 "serving claim": {func(c *Claim, _, _ *ParkScope) { c.Op = Op{Kind: KindServing, Exec: "ie-box"} }, std(moveA, "ie", "ie-box"), ErrStaleOp},
1290 "another op": {func(*Claim, *ParkScope, *ParkScope) {}, std(moveB, "ie", "ie-box"), ErrStaleOp},
1291 "moving": {func(c *Claim, _, _ *ParkScope) { c.Op.Kind, c.Op.Phase, c.Op.Exec = KindMoving, PhaseSealed, "" }, std(moveA, "ie", "ie-box"), ErrWrongState},
1292 "landing before probing": {func(c *Claim, _, _ *ParkScope) { c.Op.Phase = "restored" }, std(moveA, "ie", "ie-box"), ErrWrongState},
1293 "exec not the target": {func(c *Claim, _, _ *ParkScope) { c.Op.Exec = "ie-box" }, std(moveA, "ie", "ie-box"), ErrWrongState},
1294 "exec none": {func(c *Claim, _, _ *ParkScope) { c.Op.Exec = "" }, std(moveA, "ie", "ie-box"), ErrWrongState},
1295 "source box is not the source": {func(*Claim, *ParkScope, *ParkScope) {}, std(moveA, "ie", "uk-box"), ErrWrongState},
1296 "source region is the target": {func(*Claim, *ParkScope, *ParkScope) {}, std(moveA, "uk", "ie-box"), ErrWrongState},
1297 "source park is not the holder's location": {func(_ *Claim, s, _ *ParkScope) { s.Location = "instances/a.go.example.com/other" }, std(moveA, "ie", "ie-box"), ErrMalformed},
1298 "staging park is another lane": {func(c *Claim, _, g *ParkScope) { g.Location = "moves/" + c.Host + "/" + moveB }, std(moveA, "ie", "ie-box"), ErrMalformed},
1299 } {
1300 c := probedLanding(t)
1301 c.Inventory = map[string]InventoryEntry{"ie-box": {Layout: "flat", Paths: map[string]uint64{"/data/a": 3}}}
1302 src, stg := commitScopes(c)
1303 tc.mutate(c, &src, &stg)
1304 before := mustJSON(t, c)
1305 if err := tc.fn(c, src, stg)(c); !errors.Is(err, tc.want) {
1306 t.Fatalf("%s: got %v, want %v", name, err, tc.want)
1307 }
1308 if after := mustJSON(t, c); string(after) != string(before) {
1309 t.Fatalf("%s mutated the claim:\nbefore %s\nafter %s", name, before, after)
1310 }
1311 }
1312}
1313
1314// ---- Plan 4 Task 2: the rollback and the return (§7.8, K4, K8) ----
1315
1316const (
1317 rbA = "rb_abcdefghijklmnopqrstuvwxyz"
1318 rbB = "rb_bcdefghijklmnopqrstuvwxyza"
1319 stoppedTarget = "moves/bab/app/h/rb/stopped-target.json"
1320)
1321
1322// targetWroteLanding is probedLanding with the target's probe admitted as
1323// the writer: holder.generation and writer_box are the target's, and the
1324// last admission record is the target's writer (its predecessor the
1325// source's generation in ie). The flip at gen 7 made the target's
1326// installation stamp F = 8.
1327func targetWroteLanding(t *testing.T) *Claim {
1328 t.Helper()
1329 c := probedLanding(t)
1330 c.Holder.Generation, c.Holder.WriterBox = moveA, "uk-box"
1331 if c.Holder.Stamp != 8 {
1332 t.Fatalf("fixture: the flip's stamp is %d, want 8", c.Holder.Stamp)
1333 }
1334 return c
1335}
1336
1337// rolledBack is targetWroteLanding after the rollback's claim half.
1338func rolledBack(t *testing.T) *Claim {
1339 t.Helper()
1340 c := targetWroteLanding(t)
1341 if err := RollBack(moveA, rbA)(c); err != nil {
1342 t.Fatal(err)
1343 }
1344 return c
1345}
1346
1347func releaseScope(c *Claim) ParkScope {
1348 return ParkScope{Location: c.Holder.Location, Incarnation: c.Holder.Incarnation, EvidenceAt: time.Date(2026, 10, 6, 11, 0, 0, 0, time.UTC), RequestLimit: 2 * time.Minute}
1349}
1350
1351// released is rolledBack after the target's release, with no park-local
1352// (the target listed no paths of F).
1353func released(t *testing.T) *Claim {
1354 t.Helper()
1355 c := rolledBack(t)
1356 if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, releaseScope(c), counter())(c); err != nil {
1357 t.Fatal(err)
1358 }
1359 return c
1360}
1361
1362func validAfterWrite(t *testing.T, c *Claim) {
1363 t.Helper()
1364 d := *c
1365 d.Gen++
1366 if err := d.Validate(); err != nil {
1367 t.Fatalf("the claim after the write must validate: %v", err)
1368 }
1369}
1370
1371// The commit and the rollback race on the claim as on the record: whichever
1372// writes first, the other is refused and leaves the claim as it found it.
1373func TestRollBackRacesTheCommitOnTheClaim(t *testing.T) {
1374 newID := counter()
1375 // The commit first.
1376 c := targetWroteLanding(t)
1377 src, stg := commitScopes(c)
1378 if err := Commit(moveA, "ie", "ie-box", "uk", src, stg, newID)(c); err != nil {
1379 t.Fatal(err)
1380 }
1381 before := mustJSON(t, c)
1382 if err := RollBack(moveA, rbA)(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) {
1383 t.Fatalf("rollback after the commit: %v", err)
1384 }
1385 // The rollback first.
1386 d := targetWroteLanding(t)
1387 if err := RollBack(moveA, rbA)(d); err != nil {
1388 t.Fatal(err)
1389 }
1390 if d.Op.Kind != KindRestoring || d.Op.Phase != PhaseReleasing || d.Op.Exec != "uk-box" || d.Op.Intent["rollback"] != rbA ||
1391 d.Op.ID != moveA || len(d.Op.Participants) != 2 {
1392 t.Fatalf("op after the rollback: %+v", d.Op)
1393 }
1394 // J15: the target installation's stamp F is journalled for its release.
1395 if d.Op.Journal[JournalTargetStamp] != "8" {
1396 t.Fatalf("target stamp journal %q", d.Op.Journal[JournalTargetStamp])
1397 }
1398 if d.Holder.Stamp != 8 || d.Holder.Generation != moveA {
1399 t.Fatalf("the rollback touched the holder: %+v", d.Holder)
1400 }
1401 validAfterWrite(t, d)
1402 before = mustJSON(t, d)
1403 if err := Commit(moveA, "ie", "ie-box", "uk", src, stg, newID)(d); !errors.Is(err, ErrWrongState) || string(mustJSON(t, d)) != string(before) {
1404 t.Fatalf("commit after the rollback: %v", err)
1405 }
1406 // A replay of the same rollback is a no-op; another rollback id is refused.
1407 if err := RollBack(moveA, rbA)(d); err != nil || string(mustJSON(t, d)) != string(before) {
1408 t.Fatalf("replay: %v", err)
1409 }
1410 if err := RollBack(moveA, rbB)(d); !errors.Is(err, ErrWrongState) || string(mustJSON(t, d)) != string(before) {
1411 t.Fatalf("another rollback id: %v", err)
1412 }
1413 // Every landing phase before the commit rolls back.
1414 for _, phase := range []string{"checking", "parking", "building", "published", "local", "configured", "restored", "probing", "probed"} {
1415 e := targetWroteLanding(t)
1416 e.Op.Phase = phase
1417 if err := RollBack(moveA, rbA)(e); err != nil || e.Op.Kind != KindRestoring || e.Op.Phase != PhaseReleasing {
1418 t.Fatalf("from landing/%s: %v %+v", phase, err, e.Op)
1419 }
1420 }
1421 // Refusals leave the claim alone.
1422 for name, tc := range map[string]struct {
1423 mutate func(*Claim)
1424 op, rb string
1425 want error
1426 }{
1427 "another op": {func(*Claim) {}, moveB, rbA, ErrStaleOp},
1428 "a move id": {func(*Claim) {}, moveA, moveB, ErrMalformed},
1429 "no id": {func(*Claim) {}, moveA, "", ErrMalformed},
1430 "short id": {func(*Claim) {}, moveA, "rb_abc", ErrMalformed},
1431 "serving": {func(c *Claim) { c.Op = Op{Kind: KindServing, Exec: "uk-box"} }, moveA, rbA, ErrStaleOp},
1432 "moving, sealed": {func(c *Claim) { c.Op.Kind, c.Op.Phase, c.Op.Exec = KindMoving, PhaseSealed, "" }, moveA, rbA, ErrWrongState},
1433 "reverting": {func(c *Claim) { c.Op.Kind, c.Op.Phase, c.Op.Exec = KindReverting, "", "ie-box" }, moveA, rbA, ErrWrongState},
1434 "landing, no holder": {func(c *Claim) { c.Holder = nil }, moveA, rbA, ErrWrongState},
1435 "landing, committed": {func(c *Claim) { c.Op.Phase = PhaseCommitted }, moveA, rbA, ErrWrongState},
1436 "restoring under rbB": {func(c *Claim) { c.Op.Kind, c.Op.Phase, c.Op.Intent["rollback"] = KindRestoring, PhaseReleasing, rbB }, moveA, rbA, ErrWrongState},
1437 } {
1438 e := targetWroteLanding(t)
1439 tc.mutate(e)
1440 before := mustJSON(t, e)
1441 if err := RollBack(tc.op, tc.rb)(e); !errors.Is(err, tc.want) || string(mustJSON(t, e)) != string(before) {
1442 t.Fatalf("%s: got %v, want %v", name, err, tc.want)
1443 }
1444 }
1445}
1446
1447// J16: a record already at landing whose claim is still moving/sealed (the
1448// flip's claim half not yet written) rolls back through RollBackFlipped,
1449// which writes the flip first, in the same write.
1450func TestRollBackFlippedWritesTheFlipFirst(t *testing.T) {
1451 c := movingToSealed(t)
1452 c.Gen = 7
1453 before := mustJSON(t, c)
1454 if err := RollBack(moveA, rbA)(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) {
1455 t.Fatalf("plain rollback over moving/sealed: %v", err)
1456 }
1457 // The flip half refuses (the source's stamp is wrong): nothing is written.
1458 if err := RollBackFlipped(moveA, rbA, "uk-box", 2, "gen-7")(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) {
1459 t.Fatalf("a refused flip half: %v\nbefore %s\nafter %s", err, before, mustJSON(t, c))
1460 }
1461 if err := RollBackFlipped(moveA, "", "uk-box", 3, "gen-7")(c); !errors.Is(err, ErrMalformed) || string(mustJSON(t, c)) != string(before) {
1462 t.Fatalf("a malformed rollback id: %v", err)
1463 }
1464 if err := RollBackFlipped(moveA, rbA, "uk-box", 3, "gen-7")(c); err != nil {
1465 t.Fatal(err)
1466 }
1467 if c.Op.Kind != KindRestoring || c.Op.Phase != PhaseReleasing || c.Op.Exec != "uk-box" || c.Op.Intent["rollback"] != rbA {
1468 t.Fatalf("op: %+v", c.Op)
1469 }
1470 if c.Holder.Stamp != 8 || c.Op.Journal[JournalTargetStamp] != "8" || c.Op.Journal[JournalSourceStamp] != "3" || c.Op.Journal[JournalSrcGen] != "gen-7" {
1471 t.Fatalf("stamps: holder %d journal %v", c.Holder.Stamp, c.Op.Journal)
1472 }
1473 validAfterWrite(t, c)
1474 // Over an already flipped (landing) claim it is RollBack, and a replay is a no-op.
1475 d := targetWroteLanding(t)
1476 if err := RollBackFlipped(moveA, rbA, "uk-box", 3, "gen-7")(d); err != nil || d.Op.Kind != KindRestoring {
1477 t.Fatalf("over landing: %v %+v", err, d.Op)
1478 }
1479 after := mustJSON(t, d)
1480 if err := RollBackFlipped(moveA, rbA, "uk-box", 3, "gen-7")(d); err != nil || string(mustJSON(t, d)) != string(after) {
1481 t.Fatalf("replay: %v", err)
1482 }
1483}
1484
1485// §7.8 released, claim rev 6.14 item 6 (R12-3), J15.
1486func TestTargetReleasedCoversTheTargetRegion(t *testing.T) {
1487 build := func(inventory map[string]InventoryEntry) *Claim {
1488 c := targetWroteLanding(t)
1489 // A restarted probe: its record's predecessor is the dead probe's
1490 // generation, in the target's region.
1491 c.Holder.Admissions = append(c.Holder.Admissions, AdmissionRecord{Region: "uk", PredecessorRegion: "uk", Session: 10, Box: "uk-box", Generation: "gen-r2", Predecessor: moveA, Mode: "cold"})
1492 c.Holder.Session, c.Holder.Generation = 10, "gen-r2"
1493 c.Inventory = inventory
1494 if err := RollBack(moveA, rbA)(c); err != nil {
1495 t.Fatal(err)
1496 }
1497 return c
1498 }
1499 withF := map[string]InventoryEntry{"uk-box": {Layout: "nested", Paths: map[string]uint64{"/data/uk": 8}}}
1500 c := build(withF)
1501 ids := 0
1502 newID := func() string { ids++; return fmt.Sprintf("t%d", ids) }
1503 holder := *c.Holder
1504 if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, releaseScope(c), newID)(c); err != nil {
1505 t.Fatal(err)
1506 }
1507 if c.Op.Kind != KindRestoring || c.Op.Phase != PhaseReleased || c.Op.Exec != "" || c.Op.Journal[JournalEvidenceTarget] != stoppedTarget {
1508 t.Fatalf("op: %+v", c.Op)
1509 }
1510 k := obligationKinds(c)
1511 if k["park-replica/uk"] != 1 || k["park-local/uk-box"] != 1 || len(c.Obligations) != 2 {
1512 t.Fatalf("obligations: %+v", c.Obligations)
1513 }
1514 for _, o := range c.Obligations {
1515 if o.Op != moveA || (o.Kind == "park-replica" && (o.Park == nil || o.Park.Location != c.Holder.Location || o.Owner != "region:uk")) {
1516 t.Fatalf("obligation: %+v", o)
1517 }
1518 }
1519 if c.Holder.Admissions[0].Covered || !c.Holder.Admissions[1].Covered {
1520 t.Fatalf("coverage: ie-predecessor %v, uk-predecessor %v", c.Holder.Admissions[0].Covered, c.Holder.Admissions[1].Covered)
1521 }
1522 if c.Holder.Stamp != holder.Stamp || c.Holder.Session != holder.Session || c.Holder.Generation != holder.Generation || c.Holder.WriterBox != holder.WriterBox {
1523 t.Fatalf("the release touched the holder: %+v", c.Holder)
1524 }
1525 validAfterWrite(t, c)
1526 // A replay queues nothing and mints no ids, even once a chore is done.
1527 used, before := ids, mustJSON(t, c)
1528 if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, releaseScope(c), newID)(c); err != nil || ids != used || string(mustJSON(t, c)) != string(before) {
1529 t.Fatalf("replay: %v ids %d->%d", err, used, ids)
1530 }
1531 c.Obligations = c.Obligations[:1]
1532 before = mustJSON(t, c)
1533 if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, releaseScope(c), newID)(c); err != nil || ids != used || string(mustJSON(t, c)) != string(before) {
1534 t.Fatalf("replay after a completed chore: %v ids %d->%d", err, used, ids)
1535 }
1536 // J15: no paths of F on the target, no park-local.
1537 for name, inv := range map[string]map[string]InventoryEntry{
1538 "no inventory": nil,
1539 "another stamp": {"uk-box": {Layout: "nested", Paths: map[string]uint64{"/data/uk": 3}}},
1540 "F on another box": {"ie-box": {Layout: "nested", Paths: map[string]uint64{"/data/ie": 8}}},
1541 } {
1542 d := build(inv)
1543 if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, releaseScope(d), counter())(d); err != nil {
1544 t.Fatalf("%s: %v", name, err)
1545 }
1546 if k := obligationKinds(d); k["park-replica/uk"] != 1 || len(d.Obligations) != 1 {
1547 t.Fatalf("%s: obligations %+v", name, d.Obligations)
1548 }
1549 }
1550 // Refusals leave the claim as found.
1551 for name, tc := range map[string]struct {
1552 mutate func(*Claim)
1553 box, region, ev, op string
1554 want error
1555 }{
1556 "another op": {func(*Claim) {}, "uk-box", "uk", stoppedTarget, moveB, ErrStaleOp},
1557 "not rolled back": {func(c *Claim) { c.Op.Kind, c.Op.Phase = KindLanding, "probed" }, "uk-box", "uk", stoppedTarget, moveA, ErrWrongState},
1558 "the source's box": {func(*Claim) {}, "ie-box", "uk", stoppedTarget, moveA, ErrWrongState},
1559 "the source's region": {func(*Claim) {}, "uk-box", "ie", stoppedTarget, moveA, ErrWrongState},
1560 "no evidence": {func(*Claim) {}, "uk-box", "uk", "", moveA, ErrMalformed},
1561 "exec is not the target": {func(c *Claim) { c.Op.Exec = "ie-box" }, "uk-box", "uk", stoppedTarget, moveA, ErrWrongState},
1562 "released, other evidence": {func(c *Claim) {
1563 c.Op.Phase, c.Op.Exec, c.Op.Journal[JournalEvidenceTarget] = PhaseReleased, "", "other"
1564 }, "uk-box", "uk", stoppedTarget, moveA, ErrWrongState},
1565 } {
1566 d := build(withF)
1567 tc.mutate(d)
1568 before := mustJSON(t, d)
1569 if err := TargetReleased(tc.op, tc.box, tc.region, tc.ev, releaseScope(d), counter())(d); !errors.Is(err, tc.want) || string(mustJSON(t, d)) != string(before) {
1570 t.Fatalf("%s: got %v, want %v", name, err, tc.want)
1571 }
1572 }
1573 // A location park that is not the holder's location refuses the whole
1574 // write: exec, phase and coverage stay as found.
1575 d := build(withF)
1576 before = mustJSON(t, d)
1577 bad := releaseScope(d)
1578 bad.Location = "instances/" + d.Host
1579 if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, bad, counter())(d); !errors.Is(err, ErrMalformed) || string(mustJSON(t, d)) != string(before) {
1580 t.Fatalf("bad scope: %v\nbefore %s\nafter %s", err, before, mustJSON(t, d))
1581 }
1582}
1583
1584// Review Focus 3 (wCleanReturnIdentity, INV-16, R11-4): the clean return
1585// names the source's generation and stamp again and leaves the admissions,
1586// whose last record is the target's writer, for the re-admission to name.
1587func TestReturnToSourceCleanNamesSrcGenAndKeepsAdmissions(t *testing.T) {
1588 c := released(t)
1589 admissions := append([]AdmissionRecord(nil), c.Holder.Admissions...)
1590 session, writer := c.Holder.Session, c.Holder.WriterBox
1591 if err := ReturnToSource(moveA, "ie-box", true, ReturnFence{})(c); err != nil {
1592 t.Fatal(err)
1593 }
1594 if c.Op.Kind != KindRestoring || c.Op.Phase != PhaseClaimed || c.Op.Exec != "ie-box" {
1595 t.Fatalf("op: %+v", c.Op)
1596 }
1597 if c.Holder.Generation != "gen-7" || c.Holder.Stamp != 3 {
1598 t.Fatalf("holder generation %q stamp %d, want gen-7 and 3", c.Holder.Generation, c.Holder.Stamp)
1599 }
1600 if !reflect.DeepEqual(c.Holder.Admissions, admissions) || c.Holder.Session != session || c.Holder.WriterBox != writer {
1601 t.Fatalf("the return touched the admissions: %+v", c.Holder)
1602 }
1603 if last := c.Holder.Admissions[len(c.Holder.Admissions)-1]; last.Box != "uk-box" || last.Generation != moveA {
1604 t.Fatalf("the last admission is not the target's writer: %+v", last)
1605 }
1606 validAfterWrite(t, c)
1607 // A replay is a no-op.
1608 before := mustJSON(t, c)
1609 if err := ReturnToSource(moveA, "ie-box", true, ReturnFence{})(c); err != nil || string(mustJSON(t, c)) != string(before) {
1610 t.Fatalf("replay: %v", err)
1611 }
1612 // Refusals.
1613 for name, tc := range map[string]struct {
1614 claim func() *Claim
1615 op string
1616 source string
1617 want error
1618 }{
1619 "another op": {func() *Claim { return released(t) }, moveB, "ie-box", ErrStaleOp},
1620 "the target": {func() *Claim { return released(t) }, moveA, "uk-box", ErrWrongState},
1621 "no source": {func() *Claim { return released(t) }, moveA, "", ErrMalformed},
1622 "before the release": {func() *Claim { return rolledBack(t) }, moveA, "ie-box", ErrWrongState},
1623 "landing": {func() *Claim { return targetWroteLanding(t) }, moveA, "ie-box", ErrWrongState},
1624 "no evidence": {func() *Claim {
1625 c := released(t)
1626 delete(c.Op.Journal, JournalEvidenceTarget)
1627 return c
1628 }, moveA, "ie-box", ErrWrongState},
1629 "no source stamp": {func() *Claim {
1630 c := released(t)
1631 delete(c.Op.Journal, JournalSourceStamp)
1632 return c
1633 }, moveA, "ie-box", ErrWrongState},
1634 "no source gen": {func() *Claim {
1635 c := released(t)
1636 delete(c.Op.Journal, JournalSrcGen)
1637 return c
1638 }, moveA, "ie-box", ErrWrongState},
1639 "exec still a box": {func() *Claim {
1640 c := released(t)
1641 c.Op.Exec = "uk-box"
1642 return c
1643 }, moveA, "ie-box", ErrWrongState},
1644 "claimed by the other": {func() *Claim {
1645 c := released(t)
1646 c.Op.Phase, c.Op.Exec = PhaseClaimed, "uk-box"
1647 return c
1648 }, moveA, "ie-box", ErrWrongState},
1649 "probing": {func() *Claim {
1650 c := released(t)
1651 c.Op.Phase, c.Op.Exec = PhaseProbing, "ie-box"
1652 return c
1653 }, moveA, "ie-box", ErrWrongState},
1654 } {
1655 c := tc.claim()
1656 before := mustJSON(t, c)
1657 if err := ReturnToSource(tc.op, tc.source, true, ReturnFence{})(c); !errors.Is(err, tc.want) || string(mustJSON(t, c)) != string(before) {
1658 t.Fatalf("%s: got %v, want %v", name, err, tc.want)
1659 }
1660 }
1661}
1662
1663// A dirty return names the source's stamp again and leaves holder.generation
1664// at the target's last admission: the source's probe names the return
1665// generation at its cold admission (§7.8, §19.1 table).
1666func TestReturnToSourceDirtyLeavesGeneration(t *testing.T) {
1667 c := released(t)
1668 if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(c); err != nil {
1669 t.Fatal(err)
1670 }
1671 if c.Op.Phase != PhaseClaimed || c.Op.Exec != "ie-box" || c.Holder.Stamp != 3 || c.Holder.Generation != moveA {
1672 t.Fatalf("op %+v holder %+v", c.Op, c.Holder)
1673 }
1674 // src_gen may be the origin (""): the clean return then names it.
1675 d := released(t)
1676 d.Op.Journal[JournalSrcGen] = ""
1677 if err := ReturnToSource(moveA, "ie-box", true, ReturnFence{})(d); err != nil || d.Holder.Generation != "" {
1678 t.Fatalf("origin generation: %v %q", err, d.Holder.Generation)
1679 }
1680}
1681
1682// declaredLost is rolledBack (restoring/releasing, exec the target) after
1683// claim Plan 8's declare-lost transition for the target.
1684func declaredLost(t *testing.T) *Claim {
1685 t.Helper()
1686 c := rolledBack(t)
1687 in := LostDeclaration{LossID: "loss_1", ExpectedClaimGen: c.Gen, ExpectedClaimETag: `"etag"`,
1688 HolderIncarnation: c.Holder.Incarnation, HolderStamp: c.Holder.Stamp, HolderSession: c.Holder.Session,
1689 OpID: moveA, OpKind: KindRestoring, OpPhase: PhaseReleasing, Box: "uk-box", Enrollment: "enroll-uk",
1690 QuarantineRevision: 1, FencedAt: time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC),
1691 Consequences: LossConsequences{RevokeExecution: true, RemoveParticipation: true, RecoveryIntent: []byte(`{"kind":"lost-target"}`)}}
1692 if err := declareLostTransition(in, testLossOwner{})(c); err != nil {
1693 t.Fatal(err)
1694 }
1695 if c.Op.Exec != "none" || !reflect.DeepEqual(c.Op.Participants, []string{"ie-box"}) || c.Op.Journal["loss.loss_1"] == "" || c.Op.Journal[JournalEvidenceTarget] != "" || c.Op.Phase != PhaseReleasing {
1696 t.Fatalf("fixture: %+v", c.Op)
1697 }
1698 return c
1699}
1700
1701// §7.10: an operator's declaration stands in for the target's evidence. The
1702// claim is as Plan 8's transition leaves it (exec "none", the target out of
1703// participants, loss.<id> journalled, no evidence_target), stepped to
1704// released by the resolve door (J16) before the source claims.
1705func TestReturnToSourceAfterADeclaredLoss(t *testing.T) {
1706 c := declaredLost(t)
1707 before := mustJSON(t, c)
1708 // J16: the resolve door steps the claim to released first.
1709 if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) {
1710 t.Fatalf("at releasing: %v", err)
1711 }
1712 c.Op.Phase = PhaseReleased
1713 if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(c); err != nil {
1714 t.Fatal(err)
1715 }
1716 if c.Op.Exec != "ie-box" || c.Op.Phase != PhaseClaimed || c.Holder.Stamp != 3 {
1717 t.Fatalf("op %+v stamp %d", c.Op, c.Holder.Stamp)
1718 }
1719 validAfterWrite(t, c)
1720 // The same without the loss entry is refused.
1721 d := declaredLost(t)
1722 d.Op.Phase = PhaseReleased
1723 delete(d.Op.Journal, "loss.loss_1")
1724 before = mustJSON(t, d)
1725 if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(d); !errors.Is(err, ErrWrongState) || string(mustJSON(t, d)) != string(before) {
1726 t.Fatalf("without the loss entry: %v", err)
1727 }
1728 // A loss entry the claim never recorded, or one that left the target a
1729 // participant, is not the target's declaration.
1730 e := declaredLost(t)
1731 e.Op.Phase = PhaseReleased
1732 e.LossRefs = nil
1733 if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(e); !errors.Is(err, ErrWrongState) {
1734 t.Fatalf("an unrecorded loss: %v", err)
1735 }
1736 f := declaredLost(t)
1737 f.Op.Phase = PhaseReleased
1738 f.Op.Participants = []string{"ie-box", "uk-box"}
1739 if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(f); !errors.Is(err, ErrWrongState) {
1740 t.Fatalf("the target still a participant: %v", err)
1741 }
1742}
1743
1744// J5: a termination fence for the target, verified by the caller before
1745// Update (VerifyReturnFence), authorises the return when the closure finds
1746// the same fence.<target> entry.
1747func TestReturnToSourceWithAVerifiedFence(t *testing.T) {
1748 const ref = `{"enrollment":"0123456789abcdef0123456789abcdef","key":"control/box-fences/0123456789abcdef0123456789abcdef.json","sha256":"00"}`
1749 fenced := func() *Claim {
1750 c := rolledBack(t)
1751 c.Op.Phase, c.Op.Exec = PhaseReleased, "none"
1752 c.Op.Journal["fence.uk-box"] = ref
1753 return c
1754 }
1755 good := ReturnFence{op: moveA, box: "uk-box", ref: ref}
1756 c := fenced()
1757 if err := ReturnToSource(moveA, "ie-box", true, good)(c); err != nil {
1758 t.Fatal(err)
1759 }
1760 if c.Op.Exec != "ie-box" || c.Op.Phase != PhaseClaimed || c.Holder.Generation != "gen-7" {
1761 t.Fatalf("op %+v holder %+v", c.Op, c.Holder)
1762 }
1763 for name, f := range map[string]ReturnFence{
1764 "unverified": {},
1765 "another op": {op: moveB, box: "uk-box", ref: ref},
1766 "the source's fence": {op: moveA, box: "ie-box", ref: ref},
1767 "another journal value": {op: moveA, box: "uk-box", ref: ref + " "},
1768 } {
1769 d := fenced()
1770 before := mustJSON(t, d)
1771 if err := ReturnToSource(moveA, "ie-box", true, f)(d); !errors.Is(err, ErrWrongState) || string(mustJSON(t, d)) != string(before) {
1772 t.Fatalf("%s: %v", name, err)
1773 }
1774 }
1775 // VerifyReturnFence verifies before it vouches: a journal entry that does
1776 // not decode, or no store, is ErrFence.
1777 op := fenced().Op
1778 if _, err := VerifyReturnFence(context.Background(), nil, nil, op, "uk-box"); !errors.Is(err, ErrFence) {
1779 t.Fatalf("no store: %v", err)
1780 }
1781 op.Journal["fence.uk-box"] = "not json"
1782 if _, err := VerifyReturnFence(context.Background(), release.NewDirStore(t.TempDir()), nil, op, "uk-box"); !errors.Is(err, ErrFence) {
1783 t.Fatalf("bad journal: %v", err)
1784 }
1785}
1786
1787// K8 and J6: the release to serving from each terminal move state, once no
1788// chore under the op remains. The executor named is the one executing.
1789func TestReleaseMoveWaitsForObligations(t *testing.T) {
1790 states := map[string]struct {
1791 kind, phase, exec string
1792 }{
1793 "committed landing": {KindLanding, PhaseCommitted, "uk-box"},
1794 "opened rollback": {KindRestoring, PhaseOpened, "ie-box"},
1795 "opened revert": {KindReverting, PhaseOpened, "ie-box"},
1796 "the door, before the seal": {KindMoving, PhaseMoving, "ie-box"},
1797 }
1798 for name, st := range states {
1799 c := targetWroteLanding(t)
1800 c.Op.Kind, c.Op.Phase, c.Op.Exec = st.kind, st.phase, st.exec
1801 c.Obligations = []Obligation{{ID: "o1", Op: moveA, Kind: "park-replica", Region: "ie", Retirement: "r", Owner: "region:ie"}}
1802 before := mustJSON(t, c)
1803 if err := ReleaseMove(moveA, st.exec)(c); !errors.Is(err, ErrPendingChore) || string(mustJSON(t, c)) != string(before) {
1804 t.Fatalf("%s with a chore: %v", name, err)
1805 }
1806 c.Obligations[0].Op = "another-op"
1807 if err := ReleaseMove(moveA, st.exec)(c); err != nil {
1808 t.Fatalf("%s: %v", name, err)
1809 }
1810 if c.Op.Kind != KindServing || c.Op.Exec != st.exec || c.Op.ID != "" || c.Op.Phase != "" || len(c.Op.Participants) != 0 || c.Op.Journal["released_move"] != moveA || len(c.Op.Journal) != 1 {
1811 t.Fatalf("%s: op after release %+v", name, c.Op)
1812 }
1813 if err := RequireReleasedMove(moveA)(c); err != nil {
1814 t.Fatalf("%s: %v", name, err)
1815 }
1816 validAfterWrite(t, c)
1817 }
1818 // The new states name their executor exactly.
1819 for name, st := range states {
1820 if st.kind == KindMoving {
1821 continue
1822 }
1823 c := targetWroteLanding(t)
1824 c.Op.Kind, c.Op.Phase, c.Op.Exec = st.kind, st.phase, st.exec
1825 other := "ie-box"
1826 if st.exec == "ie-box" {
1827 other = "uk-box"
1828 }
1829 if err := ReleaseMove(moveA, other)(c); !errors.Is(err, ErrWrongState) {
1830 t.Fatalf("%s released to %s: %v", name, other, err)
1831 }
1832 }
1833 // Not before those states: a revert's phase is read from op.phase (J1),
1834 // not rev_phase.
1835 for name, st := range map[string]struct{ kind, phase string }{
1836 "landing probed": {KindLanding, "probed"},
1837 "restoring probed": {KindRestoring, PhaseProbed},
1838 "restoring claimed": {KindRestoring, PhaseClaimed},
1839 "reverting probed": {KindReverting, PhaseProbed},
1840 "reverting, rev only": {KindReverting, PhaseSealed},
1841 "moving sealed": {KindMoving, PhaseSealed},
1842 "moving aborting": {KindMoving, PhaseAborting},
1843 } {
1844 c := targetWroteLanding(t)
1845 c.Op.Kind, c.Op.Phase, c.Op.Exec = st.kind, st.phase, "ie-box"
1846 if st.phase == PhaseSealed && st.kind == KindReverting {
1847 setJournal(c, JournalRevPhase, PhaseOpened)
1848 }
1849 if err := ReleaseMove(moveA, "ie-box")(c); !errors.Is(err, ErrWrongState) {
1850 t.Fatalf("%s: %v", name, err)
1851 }
1852 }
1853}
1854
1855// revertingClaim is a revert as Plan 2's Revert leaves it: from the cut's
1856// stopping (phase "", no files baseline) or from sealed (a baseline).
1857func revertingClaim(t *testing.T, sealed bool) *Claim {
1858 t.Helper()
1859 var c *Claim
1860 if sealed {
1861 c = movingToSealed(t)
1862 } else {
1863 c = movingTestClaim(t)
1864 for _, tr := range []func(*Claim) error{
1865 BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", time.Now()),
1866 ExecToNone(moveA, "ie-box", stoppedSource, JournalEvidenceSource),
1867 } {
1868 if err := tr(c); err != nil {
1869 t.Fatal(err)
1870 }
1871 }
1872 }
1873 lane := ParkScope{Location: "moves/" + c.Host + "/" + moveA, Incarnation: c.Holder.Incarnation, EvidenceAt: time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC), RequestLimit: time.Minute}
1874 if err := Revert(moveA, "ie-box", "uk", &lane, counter())(c); err != nil {
1875 t.Fatal(err)
1876 }
1877 return c
1878}
1879
1880// §7.4 and J1, J8: rev_phase runs evidence, verify, probing, probed, opened;
1881// verify only with the files baseline the record keeps from stopped, and
1882// straight to probing without it. op.phase mirrors probing, probed and
1883// opened in the same write.
1884func TestRevPhaseOrder(t *testing.T) {
1885 for _, tc := range []struct {
1886 name string
1887 sealed bool
1888 baseline bool
1889 steps [][2]string
1890 pre string
1891 }{
1892 {"from stopped, with a baseline", true, true, [][2]string{{"evidence", "verify"}, {"verify", "probing"}, {"probing", "probed"}, {"probed", "opened"}}, PhaseSealed},
1893 {"from stopping, no baseline", false, false, [][2]string{{"evidence", "probing"}, {"probing", "probed"}, {"probed", "opened"}}, PhaseMoving},
1894 } {
1895 t.Run(tc.name, func(t *testing.T) {
1896 c := revertingClaim(t, tc.sealed)
1897 if c.Op.Journal[JournalRevPhase] != "evidence" || c.Op.Phase != tc.pre {
1898 t.Fatalf("fixture: %+v", c.Op)
1899 }
1900 // The other entry is refused.
1901 wrong := "probing"
1902 if !tc.baseline {
1903 wrong = "verify"
1904 }
1905 before := mustJSON(t, c)
1906 if err := SetRevPhase(moveA, "evidence", wrong, tc.baseline)(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) {
1907 t.Fatalf("evidence to %s: %v", wrong, err)
1908 }
1909 for _, step := range tc.steps {
1910 if err := SetRevPhase(moveA, step[0], step[1], tc.baseline)(c); err != nil {
1911 t.Fatalf("%s to %s: %v", step[0], step[1], err)
1912 }
1913 if c.Op.Journal[JournalRevPhase] != step[1] {
1914 t.Fatalf("rev_phase %q after %v", c.Op.Journal[JournalRevPhase], step)
1915 }
1916 want := tc.pre
1917 if step[1] != "verify" {
1918 want = step[1]
1919 }
1920 if c.Op.Phase != want || c.Op.Kind != KindReverting || c.Op.Exec != "ie-box" {
1921 t.Fatalf("op after %v: %+v", step, c.Op)
1922 }
1923 validAfterWrite(t, c)
1924 // A replay finds the phase moved.
1925 before := mustJSON(t, c)
1926 if err := SetRevPhase(moveA, step[0], step[1], tc.baseline)(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) {
1927 t.Fatalf("replay of %v: %v", step, err)
1928 }
1929 }
1930 // opened is the last phase; ReleaseMove takes it from there.
1931 if err := SetRevPhase(moveA, "opened", "probing", tc.baseline)(c); !errors.Is(err, ErrWrongState) {
1932 t.Fatalf("after opened: %v", err)
1933 }
1934 })
1935 }
1936 // Skips, other kinds, other ops and a phase that disagrees with rev_phase.
1937 c := revertingClaim(t, true)
1938 for name, tc := range map[string]struct {
1939 op, from, to string
1940 want error
1941 }{
1942 "skip verify to probed": {moveA, "evidence", "probed", ErrWrongState},
1943 "skip to opened": {moveA, "evidence", "opened", ErrWrongState},
1944 "from the wrong phase": {moveA, "verify", "probing", ErrWrongState},
1945 "backwards": {moveA, "evidence", "evidence", ErrWrongState},
1946 "an unknown phase": {moveA, "evidence", "stuck", ErrWrongState},
1947 "another op": {moveB, "evidence", "verify", ErrStaleOp},
1948 } {
1949 before := mustJSON(t, c)
1950 if err := SetRevPhase(tc.op, tc.from, tc.to, true)(c); !errors.Is(err, tc.want) || string(mustJSON(t, c)) != string(before) {
1951 t.Fatalf("%s: got %v, want %v", name, err, tc.want)
1952 }
1953 }
1954 d := revertingClaim(t, true)
1955 d.Op.Phase = PhaseProbed // disagrees with rev_phase evidence
1956 if err := SetRevPhase(moveA, "evidence", "verify", true)(d); !errors.Is(err, ErrWrongState) {
1957 t.Fatalf("op.phase ahead of rev_phase: %v", err)
1958 }
1959 e := rolledBack(t)
1960 setJournal(e, JournalRevPhase, "evidence")
1961 if err := SetRevPhase(moveA, "evidence", "verify", true)(e); !errors.Is(err, ErrWrongState) {
1962 t.Fatalf("a restoring op: %v", err)
1963 }
1964}
1965
1966// J1: the source's recovery phases after claimed are mirrored onto the claim
1967// op in order: probing (before the first recovery attempt), probed, opened.
1968func TestRestoringPhaseOrder(t *testing.T) {
1969 c := released(t)
1970 if err := SetRestoringPhase(moveA, PhaseReleased, PhaseClaimed)(c); !errors.Is(err, ErrWrongState) {
1971 t.Fatalf("released to claimed is ReturnToSource's: %v", err)
1972 }
1973 if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(c); err != nil {
1974 t.Fatal(err)
1975 }
1976 for _, step := range [][2]string{{PhaseClaimed, PhaseProbing}, {PhaseProbing, PhaseProbed}, {PhaseProbed, PhaseOpened}} {
1977 before := mustJSON(t, c)
1978 if err := SetRestoringPhase(moveA, step[0], "nonsense")(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) {
1979 t.Fatalf("from %s to nonsense: %v", step[0], err)
1980 }
1981 if err := SetRestoringPhase(moveA, step[0], step[1])(c); err != nil {
1982 t.Fatalf("%v: %v", step, err)
1983 }
1984 if c.Op.Phase != step[1] || c.Op.Exec != "ie-box" || c.Op.Kind != KindRestoring {
1985 t.Fatalf("op after %v: %+v", step, c.Op)
1986 }
1987 if err := SetRestoringPhase(moveA, step[0], step[1])(c); !errors.Is(err, ErrWrongState) {
1988 t.Fatalf("replay %v: %v", step, err)
1989 }
1990 }
1991 d := released(t)
1992 if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(d); err != nil {
1993 t.Fatal(err)
1994 }
1995 if err := SetRestoringPhase(moveA, PhaseClaimed, PhaseProbed)(d); !errors.Is(err, ErrWrongState) {
1996 t.Fatalf("skip: %v", err)
1997 }
1998 if err := SetRestoringPhase(moveB, PhaseClaimed, PhaseProbing)(d); !errors.Is(err, ErrStaleOp) {
1999 t.Fatalf("another op: %v", err)
2000 }
2001 // The source executes from claimed on; a claim whose exec is not the
2002 // source does not step.
2003 d.Op.Exec = "uk-box"
2004 if err := SetRestoringPhase(moveA, PhaseClaimed, PhaseProbing)(d); !errors.Is(err, ErrWrongState) {
2005 t.Fatalf("exec not the source: %v", err)
2006 }
2007}
2008
2009// §7.8 probed (dirty): the return lane moves/<host>/<rb id> is parked in the
2010// source's region, queued once, and the release to serving waits for it.
2011func TestQueueReturnLaneParkAtProbed(t *testing.T) {
2012 c := released(t)
2013 c.Obligations = nil // the target's chores are done
2014 if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(c); err != nil {
2015 t.Fatal(err)
2016 }
2017 lane := releaseScope(c)
2018 lane.Location = "moves/" + c.Host + "/" + rbA
2019 ids := 0
2020 newID := func() string { ids++; return fmt.Sprintf("l%d", ids) }
2021 for _, phase := range []string{PhaseClaimed, PhaseProbing} {
2022 c.Op.Phase = phase
2023 if err := QueueReturnLanePark(moveA, "ie", lane, newID)(c); !errors.Is(err, ErrWrongState) || ids != 0 {
2024 t.Fatalf("at %s: %v", phase, err)
2025 }
2026 }
2027 c.Op.Phase = PhaseProbed
2028 for name, tc := range map[string]struct {
2029 region string
2030 location string
2031 want error
2032 }{
2033 "the target's region": {"uk", lane.Location, ErrWrongState},
2034 "the move's lane": {"ie", "moves/" + c.Host + "/" + moveA, ErrMalformed},
2035 "the live location": {"ie", c.Holder.Location, ErrMalformed},
2036 "the staging prefix": {"ie", "moves/" + c.Host, ErrMalformed},
2037 } {
2038 s := lane
2039 s.Location = tc.location
2040 before := mustJSON(t, c)
2041 if err := QueueReturnLanePark(moveA, tc.region, s, newID)(c); !errors.Is(err, tc.want) || ids != 0 || string(mustJSON(t, c)) != string(before) {
2042 t.Fatalf("%s: %v", name, err)
2043 }
2044 }
2045 if err := QueueReturnLanePark(moveA, "ie", lane, newID)(c); err != nil {
2046 t.Fatal(err)
2047 }
2048 if k := obligationKinds(c); k["park-replica/ie"] != 1 || len(c.Obligations) != 1 || c.Obligations[0].Park.Location != lane.Location || c.Obligations[0].Owner != "region:ie" {
2049 t.Fatalf("obligations: %+v", c.Obligations)
2050 }
2051 validAfterWrite(t, c)
2052 used := ids
2053 if err := QueueReturnLanePark(moveA, "ie", lane, newID)(c); err != nil || ids != used || len(c.Obligations) != 1 {
2054 t.Fatalf("replay: %v", err)
2055 }
2056 other := lane
2057 other.RequestLimit = time.Hour
2058 if err := QueueReturnLanePark(moveA, "ie", other, newID)(c); !errors.Is(err, ErrWrongState) {
2059 t.Fatalf("another scope: %v", err)
2060 }
2061 // The release waits for the lane park.
2062 if err := SetRestoringPhase(moveA, PhaseProbed, PhaseOpened)(c); err != nil {
2063 t.Fatal(err)
2064 }
2065 if err := ReleaseMove(moveA, "ie-box")(c); !errors.Is(err, ErrPendingChore) {
2066 t.Fatalf("release before the lane park: %v", err)
2067 }
2068 // Once the park engine completes the chore, a replay at probed does not
2069 // queue it again.
2070 c.Obligations = nil
2071 c.Op.Phase = PhaseProbed
2072 if err := QueueReturnLanePark(moveA, "ie", lane, newID)(c); err != nil || len(c.Obligations) != 0 || ids != used {
2073 t.Fatalf("replay after completion: %v %+v", err, c.Obligations)
2074 }
2075 c.Op.Phase = PhaseOpened
2076 if err := ReleaseMove(moveA, "ie-box")(c); err != nil {
2077 t.Fatal(err)
2078 }
2079}
2080