| 1 | package hostclaim |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "encoding/json" |
| 6 | "errors" |
| 7 | "fmt" |
| 8 | "reflect" |
| 9 | "strings" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/release" |
| 14 | ) |
| 15 | |
| 16 | const ( |
| 17 | moveA = "mv_abcdefghijklmnopqrstuvwxyz" |
| 18 | moveB = "mv_bcdefghijklmnopqrstuvwxyza" |
| 19 | ) |
| 20 | |
| 21 | // movingTestClaim is this file's fixture (delete_test.go already defines |
| 22 | // servingClaim(t, ctx, s) with another shape). It is a VALID claim: the |
| 23 | // holder's key, incarnation and location follow the grammar Validate |
| 24 | // enforces, and the fixture asserts Validate() so it can never drift from |
| 25 | // the real grammar. |
| 26 | func movingTestClaim(t *testing.T) *Claim { |
| 27 | t.Helper() |
| 28 | created := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) |
| 29 | inc, err := Incarnation("bab", "app", "a.go.example.com", created) |
| 30 | if err != nil { |
| 31 | t.Fatal(err) |
| 32 | } |
| 33 | c := &Claim{V: schemaVersion, Host: "a.go.example.com", Gen: 3, |
| 34 | Holder: &Holder{Family: "instance", Key: "apps/bab/app/instances/a.go.example.com.json", Account: "bab", App: "app", Incarnation: inc, CreatedAt: created, Location: "instances/a.go.example.com/" + inc, Stamp: 3}, |
| 35 | Op: Op{Kind: KindServing, Exec: "ie-box"}, |
| 36 | Touched: Touched{Complete: true, Regions: []string{"ie"}, Boxes: []string{"ie-box"}}} |
| 37 | if err := c.Validate(); err != nil { |
| 38 | t.Fatalf("fixture must validate: %v", err) |
| 39 | } |
| 40 | return c |
| 41 | } |
| 42 | |
| 43 | func TestBeginMoveFromServingOnly(t *testing.T) { |
| 44 | now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) |
| 45 | c := movingTestClaim(t) |
| 46 | if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(c); err != nil { |
| 47 | t.Fatal(err) |
| 48 | } |
| 49 | if c.Op.Kind != KindMoving || c.Op.ID != moveA || c.Op.Exec != "ie-box" || c.Op.By != "ops" || !c.Op.StartedAt.Equal(now) || len(c.Op.Participants) != 2 || c.Op.Participants[0] != "ie-box" || c.Op.Participants[1] != "uk-box" || c.Op.Intent["to"] != "uk" { |
| 50 | t.Fatalf("op after BeginMove: %+v", c.Op) |
| 51 | } |
| 52 | // A plain BeginMove carries the move's own three keys and nothing else: |
| 53 | // the admission and both enrollments are BeginAdmittedMove's. |
| 54 | if want := map[string]string{"to": "uk", "source": "ie-box", "target": "uk-box"}; !reflect.DeepEqual(c.Op.Intent, want) { |
| 55 | t.Fatalf("intent after BeginMove = %v, want %v", c.Op.Intent, want) |
| 56 | } |
| 57 | if c.Op.Phase != PhaseMoving { |
| 58 | t.Fatalf("phase after BeginMove: %q", c.Op.Phase) |
| 59 | } |
| 60 | if err := c.Validate(); err != nil { |
| 61 | t.Fatalf("claim after BeginMove must validate: %v", err) |
| 62 | } |
| 63 | // K12: nothing is touched yet; the source adds the region before its |
| 64 | // first staging round. |
| 65 | if len(c.Touched.Regions) != 1 || len(c.Touched.Boxes) != 1 { |
| 66 | t.Fatalf("touched grew at BeginMove: %+v", c.Touched) |
| 67 | } |
| 68 | // Not from serving. |
| 69 | if err := BeginMove(moveB, "ops", "ie-box", "uk-box", "uk", now)(c); !errors.Is(err, ErrWrongState) { |
| 70 | t.Fatalf("second move: %v", err) |
| 71 | } |
| 72 | // exec must be the source. |
| 73 | d := movingTestClaim(t) |
| 74 | if err := BeginMove(moveA, "ops", "us-box", "uk-box", "uk", now)(d); !errors.Is(err, ErrWrongState) { |
| 75 | t.Fatalf("source not exec: %v", err) |
| 76 | } |
| 77 | // A pending chore in the target region refuses (CC11). |
| 78 | e := movingTestClaim(t) |
| 79 | e.Obligations = []Obligation{{ID: "o1", Op: "x", Kind: "park-replica", Region: "uk", Retirement: "r", Owner: "region:uk"}} |
| 80 | if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(e); !errors.Is(err, ErrPendingChore) { |
| 81 | t.Fatalf("pending chore: %v", err) |
| 82 | } |
| 83 | // ... and in the target box. |
| 84 | f := movingTestClaim(t) |
| 85 | f.Obligations = []Obligation{{ID: "o1", Op: "x", Kind: "park-local", Box: "uk-box", Retirement: "r", Owner: "box:uk-box"}} |
| 86 | if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(f); !errors.Is(err, ErrPendingChore) { |
| 87 | t.Fatalf("pending box chore: %v", err) |
| 88 | } |
| 89 | // A claim with no holder is not movable. |
| 90 | g := movingTestClaim(t) |
| 91 | g.Holder = nil |
| 92 | if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(g); !errors.Is(err, ErrWrongState) { |
| 93 | t.Fatalf("no holder: %v", err) |
| 94 | } |
| 95 | // Missing inputs are malformed. |
| 96 | for _, args := range [][4]string{{"", "ie-box", "uk-box", "uk"}, {moveA, "", "uk-box", "uk"}, {moveA, "ie-box", "", "uk"}, {moveA, "ie-box", "uk-box", ""}} { |
| 97 | h := movingTestClaim(t) |
| 98 | if err := BeginMove(args[0], "ops", args[1], args[2], args[3], now)(h); !errors.Is(err, ErrMalformed) { |
| 99 | t.Fatalf("args %v: %v", args, err) |
| 100 | } |
| 101 | if h.Op.Kind != KindServing { |
| 102 | t.Fatalf("a refused BeginMove changed the op: %+v", h.Op) |
| 103 | } |
| 104 | } |
| 105 | } |
| 106 | |
| 107 | func TestTouchAppendsOnceUnderTheOp(t *testing.T) { |
| 108 | c := movingTestClaim(t) |
| 109 | if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", time.Now())(c); err != nil { |
| 110 | t.Fatal(err) |
| 111 | } |
| 112 | if err := Touch(moveA, []string{"uk"}, nil)(c); err != nil { |
| 113 | t.Fatal(err) |
| 114 | } |
| 115 | if err := Touch(moveA, []string{"uk"}, []string{"uk-box"})(c); err != nil { |
| 116 | t.Fatal(err) |
| 117 | } |
| 118 | if len(c.Touched.Regions) != 2 || len(c.Touched.Boxes) != 2 || c.Touched.Boxes[1] != "uk-box" { |
| 119 | t.Fatalf("touched: %+v", c.Touched) |
| 120 | } |
| 121 | if err := Touch("mv_other0000000000000000000000", []string{"se"}, nil)(c); !errors.Is(err, ErrStaleOp) { |
| 122 | t.Fatalf("touch under another op: %v", err) |
| 123 | } |
| 124 | if len(c.Touched.Regions) != 2 { |
| 125 | t.Fatalf("a refused Touch changed touched: %+v", c.Touched) |
| 126 | } |
| 127 | if err := Touch("", []string{"se"}, nil)(c); !errors.Is(err, ErrStaleOp) { |
| 128 | t.Fatalf("touch under no op: %v", err) |
| 129 | } |
| 130 | if err := RequireMoveOp(moveA, KindMoving, "")(c); err != nil { |
| 131 | t.Fatal(err) |
| 132 | } |
| 133 | if err := RequireMoveOp(moveA, KindLanding, "")(c); !errors.Is(err, ErrWrongState) { |
| 134 | t.Fatalf("wrong kind: %v", err) |
| 135 | } |
| 136 | if err := RequireMoveOp(moveA, KindMoving, PhaseSealed)(c); !errors.Is(err, ErrWrongState) { |
| 137 | t.Fatalf("wrong phase: %v", err) |
| 138 | } |
| 139 | if err := RequireMoveOp(moveB, KindMoving, "")(c); !errors.Is(err, ErrStaleOp) { |
| 140 | t.Fatalf("wrong op: %v", err) |
| 141 | } |
| 142 | c.Op.Phase = PhaseSealed |
| 143 | if err := RequireMoveOp(moveA, KindMoving, PhaseSealed)(c); err != nil { |
| 144 | t.Fatalf("sealed: %v", err) |
| 145 | } |
| 146 | } |
| 147 | |
| 148 | func TestReleaseMoveFromMovingNeedsNoChores(t *testing.T) { |
| 149 | now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) |
| 150 | for _, phase := range []string{PhaseMoving} { |
| 151 | c := movingTestClaim(t) |
| 152 | if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(c); err != nil { |
| 153 | t.Fatal(err) |
| 154 | } |
| 155 | c.Op.Phase = phase |
| 156 | // A chore under this op blocks the release. |
| 157 | c.Obligations = []Obligation{{ID: "o1", Op: moveA, Kind: "park-replica", Region: "uk", Retirement: "r", Owner: "region:uk"}} |
| 158 | if err := ReleaseMove(moveA, "ie-box")(c); !errors.Is(err, ErrPendingChore) { |
| 159 | t.Fatalf("phase %q with a chore: %v", phase, err) |
| 160 | } |
| 161 | if c.Op.Kind != KindMoving { |
| 162 | t.Fatalf("a refused release changed the op: %+v", c.Op) |
| 163 | } |
| 164 | // A chore under another op does not. |
| 165 | c.Obligations[0].Op = "someone-else" |
| 166 | if err := ReleaseMove(moveB, "ie-box")(c); !errors.Is(err, ErrStaleOp) { |
| 167 | t.Fatalf("release under another op: %v", err) |
| 168 | } |
| 169 | if err := ReleaseMove(moveA, "ie-box")(c); err != nil { |
| 170 | t.Fatalf("phase %q: %v", phase, err) |
| 171 | } |
| 172 | if c.Op.Kind != KindServing || c.Op.ID != "" || c.Op.Exec != "ie-box" || c.Op.Journal["released_move"] != moveA || c.Op.Phase != "" || len(c.Op.Participants) != 0 { |
| 173 | t.Fatalf("op after release: %+v", c.Op) |
| 174 | } |
| 175 | if err := c.Validate(); err != nil { |
| 176 | t.Fatalf("claim after release must validate: %v", err) |
| 177 | } |
| 178 | } |
| 179 | // Not from an aborting move (ruling K7: that is ReleaseAbort's, after |
| 180 | // the lane park the abort queued). |
| 181 | a := movingTestClaim(t) |
| 182 | _ = BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(a) |
| 183 | a.Op.Phase = PhaseAborting |
| 184 | if err := ReleaseMove(moveA, "ie-box")(a); !errors.Is(err, ErrWrongState) { |
| 185 | t.Fatalf("release from aborting: %v", err) |
| 186 | } |
| 187 | // Not from serving, and not from a sealed move. |
| 188 | s := movingTestClaim(t) |
| 189 | if err := ReleaseMove(moveA, "ie-box")(s); !errors.Is(err, ErrStaleOp) { |
| 190 | t.Fatalf("release from serving: %v", err) |
| 191 | } |
| 192 | m := movingTestClaim(t) |
| 193 | _ = BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(m) |
| 194 | m.Op.Phase = PhaseSealed |
| 195 | if err := ReleaseMove(moveA, "ie-box")(m); !errors.Is(err, ErrWrongState) { |
| 196 | t.Fatalf("release from sealed: %v", err) |
| 197 | } |
| 198 | // Another kind under the same id is not a move. |
| 199 | l := movingTestClaim(t) |
| 200 | _ = BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(l) |
| 201 | l.Op.Kind = KindLanding |
| 202 | if err := ReleaseMove(moveA, "ie-box")(l); !errors.Is(err, ErrWrongState) { |
| 203 | t.Fatalf("release from landing: %v", err) |
| 204 | } |
| 205 | // The executor is named. |
| 206 | e := movingTestClaim(t) |
| 207 | _ = BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(e) |
| 208 | if err := ReleaseMove(moveA, "")(e); !errors.Is(err, ErrMalformed) { |
| 209 | t.Fatalf("release without exec: %v", err) |
| 210 | } |
| 211 | } |
| 212 | |
| 213 | func TestReleaseMoveLeavesEvidence(t *testing.T) { |
| 214 | now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) |
| 215 | plain := movingTestClaim(t) |
| 216 | if err := RequireReleasedMove(moveA)(plain); !errors.Is(err, ErrStaleOp) { |
| 217 | t.Fatalf("a plain serving claim satisfies the replay rule: %v", err) |
| 218 | } |
| 219 | c := movingTestClaim(t) |
| 220 | if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(c); err != nil { |
| 221 | t.Fatal(err) |
| 222 | } |
| 223 | if err := RequireReleasedMove(moveA)(c); !errors.Is(err, ErrStaleOp) { |
| 224 | t.Fatalf("a moving claim satisfies the replay rule: %v", err) |
| 225 | } |
| 226 | if err := ReleaseMove(moveA, "ie-box")(c); err != nil { |
| 227 | t.Fatal(err) |
| 228 | } |
| 229 | if err := RequireReleasedMove(moveA)(c); err != nil { |
| 230 | t.Fatalf("evidence after release: %v", err) |
| 231 | } |
| 232 | if err := RequireReleasedMove(moveB)(c); !errors.Is(err, ErrStaleOp) { |
| 233 | t.Fatalf("another move's id: %v", err) |
| 234 | } |
| 235 | if err := RequireReleasedMove("")(c); !errors.Is(err, ErrStaleOp) { |
| 236 | t.Fatalf("empty id: %v", err) |
| 237 | } |
| 238 | // A non-serving kind carrying the journal is not evidence. |
| 239 | c.Op.Kind, c.Op.ID = KindMoving, moveB |
| 240 | if err := RequireReleasedMove(moveA)(c); !errors.Is(err, ErrStaleOp) { |
| 241 | t.Fatalf("moving claim with a stale journal: %v", err) |
| 242 | } |
| 243 | } |
| 244 | |
| 245 | // BeginMove and ReleaseMove own only the op: the holder (session, writer, |
| 246 | // admissions, generation and their writes_lost), touched, inventory, |
| 247 | // obligations of other ops and the rest are carried through unchanged. |
| 248 | func TestMoveRoundTripPreservesTheRestOfTheClaim(t *testing.T) { |
| 249 | now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) |
| 250 | c := movingTestClaim(t) |
| 251 | c.Holder.Generation = "gen-7" |
| 252 | c.Holder.Session = 9 |
| 253 | c.Holder.WriterBox = "ie-box" |
| 254 | c.Holder.Admissions = []AdmissionRecord{{Region: "ie", Session: 9, Box: "ie-box", Generation: "gen-7", Mode: "admitted", Coverage: "covered", Covered: true, |
| 255 | WritesLost: &WritesLost{Box: "ie-box", Generation: "gen-6", Session: 8, RemoteMaxTXID: 41}, |
| 256 | AdditionalWritesLost: []WritesLost{{Box: "ie-box", Generation: "gen-5", Session: 7, RemoteMaxTXID: 12}}}} |
| 257 | c.Obligations = []Obligation{{ID: "o1", Op: "old", Kind: "park-replica", Region: "se", Retirement: "r", Owner: "region:se"}} |
| 258 | c.Lost = []string{"se"} |
| 259 | c.RetiredIncarnations = []string{"abc"} |
| 260 | c.QuietAfter = map[string]time.Time{"se": now} |
| 261 | c.Inventory = map[string]InventoryEntry{"ie-box": {Layout: "nested", Paths: map[string]uint64{"/data/x": 3}}} |
| 262 | if err := c.Validate(); err != nil { |
| 263 | t.Fatalf("fixture: %v", err) |
| 264 | } |
| 265 | before := mustJSON(t, c) |
| 266 | if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(c); err != nil { |
| 267 | t.Fatal(err) |
| 268 | } |
| 269 | if err := ReleaseMove(moveA, "ie-box")(c); err != nil { |
| 270 | t.Fatal(err) |
| 271 | } |
| 272 | // Only the op differs. |
| 273 | var round Claim |
| 274 | if err := json.Unmarshal(before, &round); err != nil { |
| 275 | t.Fatal(err) |
| 276 | } |
| 277 | round.Op = c.Op |
| 278 | if !reflect.DeepEqual(round, *c) { |
| 279 | t.Fatalf("round trip changed more than the op:\nbefore %s\nafter %s", before, mustJSON(t, c)) |
| 280 | } |
| 281 | if err := c.Validate(); err != nil { |
| 282 | t.Fatalf("after round trip: %v", err) |
| 283 | } |
| 284 | } |
| 285 | |
| 286 | func mustJSON(t *testing.T, c *Claim) []byte { |
| 287 | t.Helper() |
| 288 | b, err := json.Marshal(c) |
| 289 | if err != nil { |
| 290 | t.Fatal(err) |
| 291 | } |
| 292 | return b |
| 293 | } |
| 294 | |
| 295 | const stoppedSource = "moves/bab/app/h/mv/stopped-source.json" |
| 296 | |
| 297 | // movingToSealed walks a fresh moving claim to the state the flip needs: |
| 298 | // source evidence journalled, exec none, phase sealed. |
| 299 | func movingToSealed(t *testing.T) *Claim { |
| 300 | t.Helper() |
| 301 | c := movingTestClaim(t) |
| 302 | c.Holder.Generation = "gen-7" // the source's generation, which the flip must name |
| 303 | for i, tr := range []func(*Claim) error{ |
| 304 | BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", time.Now()), |
| 305 | ExecToNone(moveA, "ie-box", stoppedSource, JournalEvidenceSource), |
| 306 | SetJournal(moveA, map[string]string{JournalSeal: `{"n":7}`, JournalCutSession: "5"}), |
| 307 | SetMovePhase(moveA, KindMoving, PhaseMoving, PhaseSealed), |
| 308 | } { |
| 309 | if err := tr(c); err != nil { |
| 310 | t.Fatalf("step %d: %v", i, err) |
| 311 | } |
| 312 | } |
| 313 | return c |
| 314 | } |
| 315 | |
| 316 | func TestCutClaimCASesFollowK3AndK4(t *testing.T) { |
| 317 | c := movingTestClaim(t) |
| 318 | op := moveA |
| 319 | if err := BeginMove(op, "ops", "ie-box", "uk-box", "uk", time.Now())(c); err != nil { |
| 320 | t.Fatal(err) |
| 321 | } |
| 322 | // Only the executor gives execution up, and only citing evidence. |
| 323 | if err := ExecToNone(op, "uk-box", stoppedSource, JournalEvidenceSource)(c); !errors.Is(err, ErrWrongState) { |
| 324 | t.Fatalf("non-exec to none: %v", err) |
| 325 | } |
| 326 | if err := ExecToNone(op, "ie-box", "", JournalEvidenceSource)(c); !errors.Is(err, ErrMalformed) { |
| 327 | t.Fatalf("no evidence: %v", err) |
| 328 | } |
| 329 | if err := ExecToNone(op, "ie-box", stoppedSource, JournalSeal)(c); !errors.Is(err, ErrMalformed) { |
| 330 | t.Fatalf("evidence under a non-evidence key: %v", err) |
| 331 | } |
| 332 | if err := ExecToNone(moveB, "ie-box", stoppedSource, JournalEvidenceSource)(c); !errors.Is(err, ErrStaleOp) { |
| 333 | t.Fatalf("another op: %v", err) |
| 334 | } |
| 335 | // The flip needs the seal phase and exec none. |
| 336 | if err := Flip(op, "uk-box", 3, "")(c); !errors.Is(err, ErrWrongState) { |
| 337 | t.Fatalf("flip before sealed: %v", err) |
| 338 | } |
| 339 | if err := ExecToNone(op, "ie-box", stoppedSource, JournalEvidenceSource)(c); err != nil || c.Op.Exec != "" { |
| 340 | t.Fatalf("to none: %v %+v", err, c.Op) |
| 341 | } |
| 342 | // A crash replay of the same give-up is idempotent; other evidence is not. |
| 343 | if err := ExecToNone(op, "ie-box", stoppedSource, JournalEvidenceSource)(c); err != nil { |
| 344 | t.Fatalf("replay: %v", err) |
| 345 | } |
| 346 | if err := ExecToNone(op, "ie-box", "moves/other.json", JournalEvidenceSource)(c); !errors.Is(err, ErrWrongState) { |
| 347 | t.Fatalf("replay with other evidence: %v", err) |
| 348 | } |
| 349 | // With exec already none, an empty box is nobody: it must neither |
| 350 | // replay nor replace the evidence. |
| 351 | given := mustJSON(t, c) |
| 352 | for _, ev := range []string{stoppedSource, "moves/other.json"} { |
| 353 | if err := ExecToNone(op, "", ev, JournalEvidenceSource)(c); !errors.Is(err, ErrMalformed) { |
| 354 | t.Fatalf("empty box with %q: %v", ev, err) |
| 355 | } |
| 356 | if got := mustJSON(t, c); string(got) != string(given) { |
| 357 | t.Fatalf("empty box with %q mutated the claim: %s", ev, got) |
| 358 | } |
| 359 | } |
| 360 | // Exec none and evidence, but not sealed: still no flip. |
| 361 | if err := Flip(op, "uk-box", 3, "")(c); !errors.Is(err, ErrWrongState) { |
| 362 | t.Fatalf("flip before the seal phase: %v", err) |
| 363 | } |
| 364 | if err := SetJournal(op, map[string]string{JournalSeal: `{"n":7}`, JournalCutSession: "5"})(c); err != nil || c.Op.Journal[JournalSeal] != `{"n":7}` { |
| 365 | t.Fatalf("journal: %v", err) |
| 366 | } |
| 367 | if err := SetJournal(moveB, map[string]string{JournalSeal: "x"})(c); !errors.Is(err, ErrStaleOp) { |
| 368 | t.Fatalf("journal under another op: %v", err) |
| 369 | } |
| 370 | if err := SetMovePhase(op, KindLanding, PhaseMoving, PhaseSealed)(c); !errors.Is(err, ErrWrongState) { |
| 371 | t.Fatalf("phase under the wrong kind: %v", err) |
| 372 | } |
| 373 | if err := SetMovePhase(op, KindMoving, PhaseMoving, PhaseSealed)(c); err != nil || c.Op.Phase != PhaseSealed { |
| 374 | t.Fatalf("phase: %v", err) |
| 375 | } |
| 376 | if err := SetMovePhase(op, KindMoving, PhaseMoving, PhaseSealed)(c); !errors.Is(err, ErrWrongState) { |
| 377 | t.Fatal("phase replay") |
| 378 | } |
| 379 | // The flip's target must be the move's target. |
| 380 | if err := Flip(op, "us-box", 3, "")(c); !errors.Is(err, ErrWrongState) { |
| 381 | t.Fatalf("flip to a stranger: %v", err) |
| 382 | } |
| 383 | before := c.Holder.Stamp |
| 384 | c.Gen = 9 // what Update will have set |
| 385 | if err := Flip(op, "uk-box", before, "")(c); err != nil { |
| 386 | t.Fatal(err) |
| 387 | } |
| 388 | if c.Op.Kind != KindLanding || c.Op.Phase != "checking" || c.Op.Exec != "uk-box" || c.Op.Journal[JournalSourceStamp] != "3" || c.Op.Journal[JournalSrcGen] != "" || c.Holder.Stamp != 10 { |
| 389 | t.Fatalf("after flip: %+v stamp %d", c.Op, c.Holder.Stamp) |
| 390 | } |
| 391 | // holder.generation is untouched by the flip (section 7.5). |
| 392 | if c.Holder.Generation != "" { |
| 393 | t.Fatal("flip moved the generation") |
| 394 | } |
| 395 | } |
| 396 | |
| 397 | // Flip is written by either participant (G12): the target may write it |
| 398 | // after a source crash. A second write of the same flip is a no-op that |
| 399 | // does not restamp the holder; a different flip is refused. |
| 400 | func TestFlipIsIdempotentForEitherWriter(t *testing.T) { |
| 401 | c := movingToSealed(t) |
| 402 | c.Gen = 9 |
| 403 | // The first flip must name the source's own stamp and generation: a |
| 404 | // wrong identity would be journalled as the rollback's return. |
| 405 | sealed := mustJSON(t, c) |
| 406 | for name, fn := range map[string]func(*Claim) error{ |
| 407 | "first flip, wrong stamp": Flip(moveA, "uk-box", 4, "gen-7"), |
| 408 | "first flip, zero stamp": Flip(moveA, "uk-box", 0, "gen-7"), |
| 409 | "first flip, wrong generation": Flip(moveA, "uk-box", 3, "gen-8"), |
| 410 | "first flip, empty generation": Flip(moveA, "uk-box", 3, ""), |
| 411 | } { |
| 412 | if err := fn(c); !errors.Is(err, ErrWrongState) { |
| 413 | t.Fatalf("%s: %v", name, err) |
| 414 | } |
| 415 | if got := mustJSON(t, c); string(got) != string(sealed) { |
| 416 | t.Fatalf("%s mutated the claim:\nbefore %s\nafter %s", name, sealed, got) |
| 417 | } |
| 418 | } |
| 419 | if err := Flip(moveA, "uk-box", 3, "gen-7")(c); err != nil { |
| 420 | t.Fatal(err) |
| 421 | } |
| 422 | c.Gen = 10 // the replay observes the claim the flip produced |
| 423 | op := c.Op |
| 424 | if err := Flip(moveA, "uk-box", 3, "gen-7")(c); err != nil { |
| 425 | t.Fatalf("replay: %v", err) |
| 426 | } |
| 427 | if c.Holder.Stamp != 10 || !reflect.DeepEqual(c.Op, op) { |
| 428 | t.Fatalf("replay moved the claim: %+v stamp %d", c.Op, c.Holder.Stamp) |
| 429 | } |
| 430 | // The replay is checked against the journal, not the holder: the holder |
| 431 | // stamp is now F (10), not S (3). |
| 432 | flipped := mustJSON(t, c) |
| 433 | for name, fn := range map[string]func(*Claim) error{ |
| 434 | "another stamp": Flip(moveA, "uk-box", 4, "gen-7"), |
| 435 | "the new holder stamp": Flip(moveA, "uk-box", 10, "gen-7"), |
| 436 | "another generation": Flip(moveA, "uk-box", 3, "gen-8"), |
| 437 | "an empty generation": Flip(moveA, "uk-box", 3, ""), |
| 438 | "another target": Flip(moveA, "us-box", 3, "gen-7"), |
| 439 | "another op": Flip(moveB, "uk-box", 3, "gen-7"), |
| 440 | } { |
| 441 | if err := fn(c); err == nil { |
| 442 | t.Fatalf("%s: a different flip was accepted", name) |
| 443 | } |
| 444 | if got := mustJSON(t, c); string(got) != string(flipped) { |
| 445 | t.Fatalf("%s mutated the claim", name) |
| 446 | } |
| 447 | } |
| 448 | // Once the landing has moved past checking, the replay is stale. |
| 449 | c.Op.Phase = "committing" |
| 450 | if err := Flip(moveA, "uk-box", 3, "gen-7")(c); !errors.Is(err, ErrWrongState) { |
| 451 | t.Fatalf("replay after the landing moved on: %v", err) |
| 452 | } |
| 453 | } |
| 454 | |
| 455 | // The flip's holder.stamp is the gen the CAS produces: run it through the |
| 456 | // real Update and compare. |
| 457 | func TestFlipStampIsTheGenUpdateProduces(t *testing.T) { |
| 458 | ctx := context.Background() |
| 459 | s := release.NewDirStore(t.TempDir()) |
| 460 | c := movingToSealed(t) |
| 461 | key, err := Key(c.Host) |
| 462 | if err != nil { |
| 463 | t.Fatal(err) |
| 464 | } |
| 465 | if _, err := s.Put(ctx, key, mustJSON(t, c), ""); err != nil { |
| 466 | t.Fatal(err) |
| 467 | } |
| 468 | o, err := Read(ctx, s, c.Host) |
| 469 | if err != nil { |
| 470 | t.Fatal(err) |
| 471 | } |
| 472 | o, err = Update(ctx, s, o, Flip(moveA, "uk-box", c.Holder.Stamp, "gen-7")) |
| 473 | if err != nil { |
| 474 | t.Fatal(err) |
| 475 | } |
| 476 | if o.Claim.Holder.Stamp != o.Claim.Gen || o.Claim.Gen != c.Gen+1 { |
| 477 | t.Fatalf("stamp %d gen %d, want both %d", o.Claim.Holder.Stamp, o.Claim.Gen, c.Gen+1) |
| 478 | } |
| 479 | if o.Claim.Op.Journal[JournalSourceStamp] != "3" || o.Claim.Op.Journal[JournalSrcGen] != "gen-7" { |
| 480 | t.Fatalf("journal: %v", o.Claim.Op.Journal) |
| 481 | } |
| 482 | } |
| 483 | |
| 484 | func TestRevertNeedsEvidenceAndReturnsExecToSource(t *testing.T) { |
| 485 | c := movingTestClaim(t) |
| 486 | op := moveA |
| 487 | if err := BeginMove(op, "ops", "ie-box", "uk-box", "uk", time.Now())(c); err != nil { |
| 488 | t.Fatal(err) |
| 489 | } |
| 490 | scope := ParkScope{Location: "moves/" + c.Host + "/" + op, Incarnation: c.Holder.Incarnation, EvidenceAt: time.Now(), RequestLimit: 120 * time.Second} |
| 491 | ids := 0 |
| 492 | newID := func() string { ids++; return fmt.Sprintf("id%d", ids) } |
| 493 | if err := Revert(op, "ie-box", "uk", &scope, newID)(c); !errors.Is(err, ErrWrongState) { |
| 494 | t.Fatalf("revert while exec held: %v", err) |
| 495 | } |
| 496 | if err := ExecToNone(op, "ie-box", stoppedSource, JournalEvidenceSource)(c); err != nil { |
| 497 | t.Fatal(err) |
| 498 | } |
| 499 | if err := Revert(op, "uk-box", "uk", &scope, newID)(c); !errors.Is(err, ErrWrongState) { |
| 500 | t.Fatalf("revert to the target: %v", err) |
| 501 | } |
| 502 | if err := Revert(op, "ie-box", "se", &scope, newID)(c); !errors.Is(err, ErrWrongState) { |
| 503 | t.Fatalf("revert naming a region the move never targeted: %v", err) |
| 504 | } |
| 505 | if err := Revert(op, "ie-box", "uk", nil, newID)(c); !errors.Is(err, ErrMalformed) { |
| 506 | t.Fatalf("revert with no staging scope: %v", err) |
| 507 | } |
| 508 | live := scope |
| 509 | live.Location = c.Holder.Location |
| 510 | if err := Revert(op, "ie-box", "uk", &live, newID)(c); !errors.Is(err, ErrMalformed) { |
| 511 | t.Fatalf("revert whose park would reach the live location: %v", err) |
| 512 | } |
| 513 | // The scope is this move's own lane and this holder's incarnation: not |
| 514 | // the host-wide staging prefix, not another op's lane, not another |
| 515 | // installation's incarnation. |
| 516 | hostWide := scope |
| 517 | hostWide.Location = "moves/" + c.Host |
| 518 | otherOp := scope |
| 519 | otherOp.Location = "moves/" + c.Host + "/" + moveB |
| 520 | otherInc := scope |
| 521 | otherInc.Incarnation = strings.Repeat("ab", 32) |
| 522 | preRevert := mustJSON(t, c) |
| 523 | for name, sc := range map[string]ParkScope{"host-wide": hostWide, "another op's lane": otherOp, "another incarnation": otherInc} { |
| 524 | sc := sc |
| 525 | if err := Revert(op, "ie-box", "uk", &sc, newID)(c); !errors.Is(err, ErrMalformed) { |
| 526 | t.Fatalf("revert with a %s scope: %v", name, err) |
| 527 | } |
| 528 | if got := mustJSON(t, c); string(got) != string(preRevert) { |
| 529 | t.Fatalf("a refused revert (%s scope) mutated the claim", name) |
| 530 | } |
| 531 | } |
| 532 | if ids != 0 { |
| 533 | t.Fatalf("a refused revert allocated %d ids", ids) |
| 534 | } |
| 535 | if c.Op.Kind != KindMoving || len(c.Obligations) != 0 { |
| 536 | t.Fatalf("a refused revert left changes: %+v", c.Op) |
| 537 | } |
| 538 | if err := Revert(op, "ie-box", "uk", &scope, newID)(c); err != nil || c.Op.Kind != KindReverting || c.Op.Exec != "ie-box" || c.Op.Journal[JournalRevPhase] != "evidence" { |
| 539 | t.Fatalf("revert: %v %+v", err, c.Op) |
| 540 | } |
| 541 | if len(c.Obligations) != 1 || c.Obligations[0].Kind != "park-replica" || c.Obligations[0].Region != "uk" || c.Obligations[0].Park == nil || |
| 542 | c.Obligations[0].Owner != "region:uk" || c.Obligations[0].Op != op || c.Obligations[0].Park.Location != scope.Location { |
| 543 | t.Fatalf("revert queued no target staging park: %+v", c.Obligations) |
| 544 | } |
| 545 | c.Gen = 9 |
| 546 | if err := c.Validate(); err != nil { |
| 547 | t.Fatalf("claim after revert must validate: %v", err) |
| 548 | } |
| 549 | } |
| 550 | |
| 551 | // A revert keeps rev_phase if a replay already set it, and queues the |
| 552 | // staging park once even when one is already owed under the op. |
| 553 | func TestRevertKeepsRevPhaseAndQueuesTheParkOnce(t *testing.T) { |
| 554 | c := movingTestClaim(t) |
| 555 | if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", time.Now())(c); err != nil { |
| 556 | t.Fatal(err) |
| 557 | } |
| 558 | if err := ExecToNone(moveA, "ie-box", stoppedSource, JournalEvidenceSource)(c); err != nil { |
| 559 | t.Fatal(err) |
| 560 | } |
| 561 | scope := ParkScope{Location: "moves/" + c.Host + "/" + moveA, Incarnation: c.Holder.Incarnation, EvidenceAt: time.Now(), RequestLimit: time.Minute} |
| 562 | c.Obligations = []Obligation{{ID: "o1", Op: moveA, Kind: "park-replica", Region: "uk", Retirement: "r1", Owner: "region:uk", Park: &scope}} |
| 563 | if err := SetJournal(moveA, map[string]string{JournalRevPhase: "staging"})(c); err != nil { |
| 564 | t.Fatal(err) |
| 565 | } |
| 566 | if err := Revert(moveA, "ie-box", "uk", &scope, counter())(c); err != nil { |
| 567 | t.Fatal(err) |
| 568 | } |
| 569 | if c.Op.Journal[JournalRevPhase] != "staging" || len(c.Obligations) != 1 { |
| 570 | t.Fatalf("rev_phase %q, obligations %+v", c.Op.Journal[JournalRevPhase], c.Obligations) |
| 571 | } |
| 572 | } |
| 573 | |
| 574 | // The cut's transitions own only the op, the journal, the revert's staging |
| 575 | // park and (the flip) the holder's stamp; every other field is carried |
| 576 | // through unchanged. |
| 577 | func TestCutTransitionsPreserveTheRestOfTheClaim(t *testing.T) { |
| 578 | now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) |
| 579 | fixture := func() *Claim { |
| 580 | c := movingTestClaim(t) |
| 581 | c.Holder.Generation = "gen-7" |
| 582 | c.Holder.Session = 9 |
| 583 | c.Holder.WriterBox = "ie-box" |
| 584 | c.Holder.Admissions = []AdmissionRecord{{Region: "ie", Session: 9, Box: "ie-box", Generation: "gen-7", Mode: "admitted", Coverage: "covered", Covered: true, |
| 585 | WritesLost: &WritesLost{Box: "ie-box", Generation: "gen-6", Session: 8, RemoteMaxTXID: 41}, |
| 586 | AdditionalWritesLost: []WritesLost{{Box: "ie-box", Generation: "gen-5", Session: 7, RemoteMaxTXID: 12}}}} |
| 587 | c.Obligations = []Obligation{{ID: "o1", Op: "old", Kind: "park-replica", Region: "se", Retirement: "r", Owner: "region:se"}} |
| 588 | c.Lost = []string{"se"} |
| 589 | c.RetiredIncarnations = []string{"abc"} |
| 590 | c.QuietAfter = map[string]time.Time{"se": now} |
| 591 | c.Inventory = map[string]InventoryEntry{"ie-box": {Layout: "nested", Paths: map[string]uint64{"/data/x": 3}}} |
| 592 | if err := c.Validate(); err != nil { |
| 593 | t.Fatalf("fixture: %v", err) |
| 594 | } |
| 595 | return c |
| 596 | } |
| 597 | scope := ParkScope{Location: "moves/a.go.example.com/" + moveA, Incarnation: fixture().Holder.Incarnation, EvidenceAt: now, RequestLimit: time.Minute} |
| 598 | begin := func() []func(*Claim) error { |
| 599 | return []func(*Claim) error{ |
| 600 | BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now), |
| 601 | ExecToNone(moveA, "ie-box", stoppedSource, JournalEvidenceSource), |
| 602 | } |
| 603 | } |
| 604 | cases := map[string]struct { |
| 605 | steps []func(*Claim) error |
| 606 | restamp bool // the flip restamps the holder |
| 607 | parks int // staging parks the transition queues |
| 608 | }{ |
| 609 | "flip": {restamp: true, steps: append(begin(), |
| 610 | SetJournal(moveA, map[string]string{JournalSeal: `{"n":7}`}), |
| 611 | SetMovePhase(moveA, KindMoving, PhaseMoving, PhaseSealed), |
| 612 | Flip(moveA, "uk-box", 3, "gen-7"))}, |
| 613 | "revert": {parks: 1, steps: append(begin(), Revert(moveA, "ie-box", "uk", &scope, counter()))}, |
| 614 | } |
| 615 | for name, tc := range cases { |
| 616 | t.Run(name, func(t *testing.T) { |
| 617 | c := fixture() |
| 618 | before := mustJSON(t, c) |
| 619 | for i, tr := range tc.steps { |
| 620 | if err := tr(c); err != nil { |
| 621 | t.Fatalf("step %d: %v", i, err) |
| 622 | } |
| 623 | } |
| 624 | if len(c.Obligations) != 1+tc.parks || !reflect.DeepEqual(c.Obligations[0], fixture().Obligations[0]) { |
| 625 | t.Fatalf("obligations of other ops changed: %+v", c.Obligations) |
| 626 | } |
| 627 | var round Claim |
| 628 | if err := json.Unmarshal(before, &round); err != nil { |
| 629 | t.Fatal(err) |
| 630 | } |
| 631 | round.Op, round.Obligations = c.Op, c.Obligations |
| 632 | if tc.restamp { |
| 633 | round.Holder.Stamp = c.Holder.Stamp |
| 634 | } |
| 635 | if !reflect.DeepEqual(round, *c) { |
| 636 | t.Fatalf("%s changed more than it owns:\nbefore %s\nafter %s", name, before, mustJSON(t, c)) |
| 637 | } |
| 638 | c.Gen++ // what Update would do |
| 639 | if err := c.Validate(); err != nil { |
| 640 | t.Fatalf("after %s: %v", name, err) |
| 641 | } |
| 642 | }) |
| 643 | } |
| 644 | } |
| 645 | |
| 646 | // A revert replayed after a crash finds the claim already reverting with |
| 647 | // the same op, region, scope and rev_phase: nil, no new ids, one park. |
| 648 | func TestRevertReplayIsIdempotent(t *testing.T) { |
| 649 | c := movingTestClaim(t) |
| 650 | if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", time.Now())(c); err != nil { |
| 651 | t.Fatal(err) |
| 652 | } |
| 653 | if err := ExecToNone(moveA, "ie-box", stoppedSource, JournalEvidenceSource)(c); err != nil { |
| 654 | t.Fatal(err) |
| 655 | } |
| 656 | scope := ParkScope{Location: "moves/" + c.Host + "/" + moveA, Incarnation: c.Holder.Incarnation, EvidenceAt: time.Now().UTC(), RequestLimit: time.Minute} |
| 657 | ids := 0 |
| 658 | newID := func() string { ids++; return fmt.Sprintf("id%d", ids) } |
| 659 | if err := Revert(moveA, "ie-box", "uk", &scope, newID)(c); err != nil { |
| 660 | t.Fatal(err) |
| 661 | } |
| 662 | after, usedIDs := mustJSON(t, c), ids |
| 663 | if err := Revert(moveA, "ie-box", "uk", &scope, newID)(c); err != nil { |
| 664 | t.Fatalf("replay: %v", err) |
| 665 | } |
| 666 | if got := mustJSON(t, c); string(got) != string(after) || ids != usedIDs || len(c.Obligations) != 1 { |
| 667 | t.Fatalf("replay changed the claim or allocated ids (%d, was %d): %s", ids, usedIDs, mustJSON(t, c)) |
| 668 | } |
| 669 | // A replay that is not the same revert is refused, and changes nothing. |
| 670 | later := scope |
| 671 | later.EvidenceAt = scope.EvidenceAt.Add(time.Second) |
| 672 | for name, fn := range map[string]func(*Claim) error{ |
| 673 | "another scope": Revert(moveA, "ie-box", "uk", &later, newID), |
| 674 | "another region": Revert(moveA, "ie-box", "se", &scope, newID), |
| 675 | "another source": Revert(moveA, "uk-box", "uk", &scope, newID), |
| 676 | "another op": Revert(moveB, "ie-box", "uk", &scope, newID), |
| 677 | } { |
| 678 | if err := fn(c); err == nil { |
| 679 | t.Fatalf("%s: accepted as a replay", name) |
| 680 | } |
| 681 | if got := mustJSON(t, c); string(got) != string(after) || ids != usedIDs { |
| 682 | t.Fatalf("%s changed the claim or allocated ids", name) |
| 683 | } |
| 684 | } |
| 685 | // Once the revert has moved past evidence, the replay is stale. |
| 686 | c.Op.Journal[JournalRevPhase] = "opened" |
| 687 | if err := Revert(moveA, "ie-box", "uk", &scope, newID)(c); !errors.Is(err, ErrWrongState) { |
| 688 | t.Fatalf("replay after the revert moved on: %v", err) |
| 689 | } |
| 690 | } |
| 691 | |
| 692 | // parkFixture is a valid landing-shaped claim for the park queue helpers: |
| 693 | // op moveA, a holder with an incarnation, and the scope helpers an op needs. |
| 694 | func parkFixture(t *testing.T) (*Claim, func(location string) ParkScope) { |
| 695 | t.Helper() |
| 696 | c := movingTestClaim(t) |
| 697 | if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC))(c); err != nil { |
| 698 | t.Fatal(err) |
| 699 | } |
| 700 | scope := func(location string) ParkScope { |
| 701 | return ParkScope{Location: location, Incarnation: c.Holder.Incarnation, EvidenceAt: time.Date(2026, 10, 6, 9, 5, 0, 0, time.UTC), RequestLimit: 2 * time.Minute} |
| 702 | } |
| 703 | return c, scope |
| 704 | } |
| 705 | |
| 706 | // inState puts a claim in the op state a park helper's lifecycle check reads: |
| 707 | // the kind and phase, as the transitions that own them would leave it. |
| 708 | func inState(c *Claim, kind, phase string) { |
| 709 | c.Op.Kind, c.Op.Phase = kind, phase |
| 710 | } |
| 711 | |
| 712 | // The move's retirements ride the engine's existing kinds. A real store |
| 713 | // round trip: Validate accepts what the helper wrote and it reads back. |
| 714 | func TestLocationParkIsAParkReplicaTheStoreAccepts(t *testing.T) { |
| 715 | ctx := context.Background() |
| 716 | s := release.NewDirStore(t.TempDir()) |
| 717 | o := servingClaim(t, ctx, s) |
| 718 | now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) |
| 719 | o, err := Update(ctx, s, o, BeginMove(moveA, "ops", "adi-knitting", "uk-box", "uk", now)) |
| 720 | if err != nil { |
| 721 | t.Fatal(err) |
| 722 | } |
| 723 | inc := o.Claim.Holder.Incarnation |
| 724 | scope := ParkScope{Location: "instances/" + testHost + "/" + inc, Incarnation: inc, EvidenceAt: now, RequestLimit: 2 * time.Minute} |
| 725 | // The commit's claim state: the source's location is parked from here. |
| 726 | if o, err = Update(ctx, s, o, func(c *Claim) error { inState(c, KindLanding, "committed"); return nil }); err != nil { |
| 727 | t.Fatal(err) |
| 728 | } |
| 729 | o, err = Update(ctx, s, o, QueueLocationPark(moveA, "ie", "adi-knitting", scope, counter())) |
| 730 | if err != nil { |
| 731 | t.Fatal(err) |
| 732 | } |
| 733 | o, err = Read(ctx, s, testHost) |
| 734 | if err != nil { |
| 735 | t.Fatal(err) |
| 736 | } |
| 737 | if err := o.Claim.Validate(); err != nil { |
| 738 | t.Fatal(err) |
| 739 | } |
| 740 | if len(o.Claim.Obligations) != 2 { |
| 741 | t.Fatalf("obligations: %+v", o.Claim.Obligations) |
| 742 | } |
| 743 | r, l := o.Claim.Obligations[0], o.Claim.Obligations[1] |
| 744 | if r.Kind != "park-replica" || r.Op != moveA || r.Region != "ie" || r.Owner != "region:ie" || r.Park == nil || !sameParkScope(*r.Park, scope) || r.Retirement == "" || r.ID == "" || r.ID == r.Retirement { |
| 745 | t.Fatalf("replica park: %+v", r) |
| 746 | } |
| 747 | if l.Kind != "park-local" || l.Op != moveA || l.Box != "adi-knitting" || l.Owner != "box:adi-knitting" || l.Park != nil || l.Region != "" || l.Retirement == "" { |
| 748 | t.Fatalf("local park: %+v", l) |
| 749 | } |
| 750 | // The same call again, across the store: nothing new, no ids spent. |
| 751 | ids := 0 |
| 752 | again := func() string { ids++; return fmt.Sprintf("late_%d", ids) } |
| 753 | if o, err = Update(ctx, s, o, QueueLocationPark(moveA, "ie", "adi-knitting", scope, again)); err != nil || ids != 0 || len(o.Claim.Obligations) != 2 { |
| 754 | t.Fatalf("replay: ids=%d obligations=%d err=%v", ids, len(o.Claim.Obligations), err) |
| 755 | } |
| 756 | // Completing both by the existing chore completion ends them. |
| 757 | for _, ob := range o.Claim.Obligations { |
| 758 | if o, err = Update(ctx, s, o, CompleteObligation(ob.Op, ob.ID)); err != nil { |
| 759 | t.Fatal(err) |
| 760 | } |
| 761 | } |
| 762 | if len(o.Claim.Obligations) != 0 { |
| 763 | t.Fatal("chores left") |
| 764 | } |
| 765 | } |
| 766 | |
| 767 | func TestStagingParkScopeIsTheLane(t *testing.T) { |
| 768 | c, scope := parkFixture(t) |
| 769 | lane := scope("moves/" + c.Host + "/" + moveA) |
| 770 | if err := QueueStagingPark(moveA, "uk", lane, counter())(c); err != nil { |
| 771 | t.Fatal(err) |
| 772 | } |
| 773 | if len(c.Obligations) != 1 { |
| 774 | t.Fatalf("obligations: %+v", c.Obligations) |
| 775 | } |
| 776 | o := c.Obligations[0] |
| 777 | if o.Kind != "park-replica" || o.Region != "uk" || o.Owner != "region:uk" || o.Op != moveA || o.Park.Location != "moves/"+c.Host+"/"+moveA { |
| 778 | t.Fatalf("staging park: %+v", o) |
| 779 | } |
| 780 | c.Gen++ |
| 781 | if err := c.Validate(); err != nil { |
| 782 | t.Fatal(err) |
| 783 | } |
| 784 | // Anything but this move's own lane is refused, and nothing is queued. |
| 785 | for name, location := range map[string]string{ |
| 786 | "the live location": "instances/" + c.Host + "/" + c.Holder.Incarnation, |
| 787 | "the legacy prefix": "instances/" + c.Host, |
| 788 | "the staging prefix": "moves/" + c.Host, |
| 789 | "another move": "moves/" + c.Host + "/" + moveB, |
| 790 | "another host": "moves/other.example.com/" + moveA, |
| 791 | "below the lane": "moves/" + c.Host + "/" + moveA + "/x", |
| 792 | } { |
| 793 | d, _ := parkFixture(t) |
| 794 | bad := scope(location) |
| 795 | ids := 0 |
| 796 | if err := QueueStagingPark(moveA, "uk", bad, func() string { ids++; return "x" })(d); err == nil || len(d.Obligations) != 0 || ids != 0 { |
| 797 | t.Errorf("%s: accepted (%v), obligations %d", name, err, len(d.Obligations)) |
| 798 | } |
| 799 | } |
| 800 | // Another holder's incarnation is refused. |
| 801 | d, _ := parkFixture(t) |
| 802 | foreign := scope("moves/" + d.Host + "/" + moveA) |
| 803 | foreign.Incarnation = strings.Repeat("a", 64) |
| 804 | if err := QueueStagingPark(moveA, "uk", foreign, counter())(d); !errors.Is(err, ErrMalformed) { |
| 805 | t.Errorf("foreign incarnation: %v", err) |
| 806 | } |
| 807 | // A stale op is refused. |
| 808 | if err := QueueStagingPark(moveB, "uk", lane, counter())(d); !errors.Is(err, ErrStaleOp) { |
| 809 | t.Errorf("stale op: %v", err) |
| 810 | } |
| 811 | } |
| 812 | |
| 813 | // A revert already queued the lane's park: queueing it again is a no-op, |
| 814 | // and a different scope for the same lane is refused, not overwritten. |
| 815 | func TestStagingParkAfterRevertIsTheSameChore(t *testing.T) { |
| 816 | c, scope := parkFixture(t) |
| 817 | if err := ExecToNone(moveA, "ie-box", stoppedSource, JournalEvidenceSource)(c); err != nil { |
| 818 | t.Fatal(err) |
| 819 | } |
| 820 | lane := scope("moves/" + c.Host + "/" + moveA) |
| 821 | if err := Revert(moveA, "ie-box", "uk", &lane, counter())(c); err != nil { |
| 822 | t.Fatal(err) |
| 823 | } |
| 824 | ids := 0 |
| 825 | newID := func() string { ids++; return "z" } |
| 826 | if err := QueueStagingPark(moveA, "uk", lane, newID)(c); err != nil || ids != 0 || len(c.Obligations) != 1 { |
| 827 | t.Fatalf("same lane: %v ids=%d obligations=%d", err, ids, len(c.Obligations)) |
| 828 | } |
| 829 | later := lane |
| 830 | later.EvidenceAt = lane.EvidenceAt.Add(time.Second) |
| 831 | if err := QueueStagingPark(moveA, "uk", later, newID)(c); !errors.Is(err, ErrWrongState) || ids != 0 || len(c.Obligations) != 1 { |
| 832 | t.Fatalf("changed scope: %v ids=%d", err, ids) |
| 833 | } |
| 834 | } |
| 835 | |
| 836 | func TestLocationParkAndStagingParkCoexistInOneRegion(t *testing.T) { |
| 837 | // A rolled-back target at cleaning owes the lane and the location in the |
| 838 | // same region under one op. |
| 839 | c, scope := parkFixture(t) |
| 840 | inState(c, KindRestoring, "released") |
| 841 | lane := scope("moves/" + c.Host + "/" + moveA) |
| 842 | loc := scope(c.Holder.Location) |
| 843 | n := counter() |
| 844 | for i := 0; i < 2; i++ { |
| 845 | if err := QueueStagingPark(moveA, "uk", lane, n)(c); err != nil { |
| 846 | t.Fatal(err) |
| 847 | } |
| 848 | if err := QueueLocationPark(moveA, "uk", "", loc, n)(c); err != nil { |
| 849 | t.Fatal(err) |
| 850 | } |
| 851 | } |
| 852 | if len(c.Obligations) != 2 { |
| 853 | t.Fatalf("obligations: %+v", c.Obligations) |
| 854 | } |
| 855 | c.Gen++ |
| 856 | if err := c.Validate(); err != nil { |
| 857 | t.Fatal(err) |
| 858 | } |
| 859 | // The location's local chore is per box, not per region. |
| 860 | if err := QueueLocationPark(moveA, "uk", "uk-box", loc, n)(c); err != nil || len(c.Obligations) != 3 { |
| 861 | t.Fatalf("add the box: %v %+v", err, c.Obligations) |
| 862 | } |
| 863 | if err := QueueLocationPark(moveA, "uk", "uk-box", loc, n)(c); err != nil || len(c.Obligations) != 3 { |
| 864 | t.Fatalf("box twice: %v %+v", err, c.Obligations) |
| 865 | } |
| 866 | } |
| 867 | |
| 868 | // A whole location is retired only where the region has stopped being |
| 869 | // authoritative: the source after the commit (landing/committed), and a |
| 870 | // rolled-back target (restoring, from its release). Every other state, and |
| 871 | // the other region in either state, is refused with nothing queued and no |
| 872 | // ids spent. |
| 873 | func TestLocationParkOnlyWhereTheRegionIsRetired(t *testing.T) { |
| 874 | type tc struct { |
| 875 | kind, phase, region, box string |
| 876 | allowed bool |
| 877 | } |
| 878 | cases := map[string]tc{ |
| 879 | "source after the commit": {KindLanding, "committed", "ie", "ie-box", true}, |
| 880 | "source after the commit, region only": {KindLanding, "committed", "ie", "", true}, |
| 881 | "rolled-back target at cleaning": {KindRestoring, "released", "uk", "uk-box", true}, |
| 882 | "rolled-back target after the source": {KindRestoring, "claimed", "uk", "uk-box", true}, |
| 883 | "source before the cut": {KindMoving, PhaseMoving, "ie", "ie-box", false}, |
| 884 | "source sealed": {KindMoving, PhaseSealed, "ie", "ie-box", false}, |
| 885 | "source in the abort": {KindMoving, PhaseAborting, "ie", "ie-box", false}, |
| 886 | "source while the landing checks": {KindLanding, "checking", "ie", "ie-box", false}, |
| 887 | "source while the landing probes": {KindLanding, "probing", "ie", "ie-box", false}, |
| 888 | "source while reverting": {KindReverting, "evidence", "ie", "ie-box", false}, |
| 889 | "source after the rollback began": {KindRestoring, "released", "ie", "ie-box", false}, |
| 890 | "target after the commit": {KindLanding, "committed", "uk", "uk-box", false}, |
| 891 | "target before the commit": {KindLanding, "probing", "uk", "uk-box", false}, |
| 892 | "target before its release": {KindRestoring, "releasing", "uk", "uk-box", false}, |
| 893 | "target still serving": {KindServing, "", "uk", "uk-box", false}, |
| 894 | "the target's box from the source": {KindLanding, "committed", "ie", "uk-box", false}, |
| 895 | "the source's box from a rolled-back uk": {KindRestoring, "released", "uk", "ie-box", false}, |
| 896 | } |
| 897 | for name, tc := range cases { |
| 898 | t.Run(name, func(t *testing.T) { |
| 899 | c, scope := parkFixture(t) |
| 900 | inState(c, tc.kind, tc.phase) |
| 901 | ids := 0 |
| 902 | newID := func() string { ids++; return fmt.Sprintf("p%d", ids) } |
| 903 | before := mustJSON(t, c) |
| 904 | err := QueueLocationPark(moveA, tc.region, tc.box, scope(c.Holder.Location), newID)(c) |
| 905 | if tc.allowed { |
| 906 | if err != nil || len(c.Obligations) == 0 { |
| 907 | t.Fatalf("refused: %v", err) |
| 908 | } |
| 909 | c.Gen++ |
| 910 | if err := c.Validate(); err != nil { |
| 911 | t.Fatal(err) |
| 912 | } |
| 913 | return |
| 914 | } |
| 915 | if !errors.Is(err, ErrWrongState) || ids != 0 || string(mustJSON(t, c)) != string(before) { |
| 916 | t.Fatalf("err=%v ids=%d, claim changed=%v", err, ids, string(mustJSON(t, c)) != string(before)) |
| 917 | } |
| 918 | }) |
| 919 | } |
| 920 | } |
| 921 | |
| 922 | // Location and residue parks name holder.location, whichever form the |
| 923 | // holder uses (spec §6.4): the legacy instances/<host> or the |
| 924 | // incarnation-scoped instances/<host>/<inc>, never the other one. A trailing |
| 925 | // slash is normalised away on both sides. |
| 926 | func TestParkLocationMustBeTheHoldersLocation(t *testing.T) { |
| 927 | for _, form := range []string{"scoped", "legacy"} { |
| 928 | t.Run(form, func(t *testing.T) { |
| 929 | c, scope := parkFixture(t) |
| 930 | scoped := "instances/" + c.Host + "/" + c.Holder.Incarnation |
| 931 | legacy := "instances/" + c.Host |
| 932 | own, other := scoped, legacy |
| 933 | if form == "legacy" { |
| 934 | own, other = legacy, scoped |
| 935 | } |
| 936 | c.Holder.Location = own |
| 937 | inState(c, KindLanding, "committed") |
| 938 | if err := c.Validate(); err != nil { |
| 939 | t.Fatal(err) |
| 940 | } |
| 941 | ids := 0 |
| 942 | newID := func() string { ids++; return fmt.Sprintf("p%d", ids) } |
| 943 | before := mustJSON(t, c) |
| 944 | for name, fn := range map[string]func(*Claim) error{ |
| 945 | "location": QueueLocationPark(moveA, "ie", "ie-box", scope(other), newID), |
| 946 | "residue": QueueResidueParks(moveA, "ie", scope(other), []string{"gen-1"}, newID), |
| 947 | "slash/other": QueueLocationPark(moveA, "ie", "", scope(other+"/"), newID), |
| 948 | } { |
| 949 | if err := fn(c); !errors.Is(err, ErrMalformed) && !errors.Is(err, ErrWrongState) || ids != 0 || string(mustJSON(t, c)) != string(before) { |
| 950 | t.Errorf("%s: err=%v ids=%d", name, err, ids) |
| 951 | } |
| 952 | } |
| 953 | // The holder's own form is accepted, with or without a trailing slash, |
| 954 | // and what is stored has none. |
| 955 | if err := QueueLocationPark(moveA, "ie", "", scope(own+"/"), newID)(c); err != nil { |
| 956 | t.Fatalf("own location with a slash: %v", err) |
| 957 | } |
| 958 | if err := QueueResidueParks(moveA, "ie", scope(own), []string{"gen-1"}, newID)(c); err != nil { |
| 959 | t.Fatalf("own location residue: %v", err) |
| 960 | } |
| 961 | if len(c.Obligations) != 2 || c.Obligations[0].Park.Location != own || c.Obligations[1].Park.Location != own { |
| 962 | t.Fatalf("stored: %+v", c.Obligations) |
| 963 | } |
| 964 | }) |
| 965 | } |
| 966 | } |
| 967 | |
| 968 | func TestLocationParkRefusals(t *testing.T) { |
| 969 | c, scope := parkFixture(t) |
| 970 | inState(c, KindLanding, "committed") |
| 971 | loc := scope(c.Holder.Location) |
| 972 | lane := scope("moves/" + c.Host + "/" + moveA) |
| 973 | bad := loc |
| 974 | bad.RequestLimit = 0 |
| 975 | changed := loc |
| 976 | changed.EvidenceAt = loc.EvidenceAt.Add(time.Hour) |
| 977 | if err := QueueLocationPark(moveA, "ie", "", loc, counter())(c); err != nil { |
| 978 | t.Fatal(err) |
| 979 | } |
| 980 | for name, fn := range map[string]func(*Claim) error{ |
| 981 | "a lane through the location door": QueueLocationPark(moveA, "ie", "", lane, counter()), |
| 982 | "no region": QueueLocationPark(moveA, "", "", loc, counter()), |
| 983 | "an invalid scope": QueueLocationPark(moveA, "ie", "", bad, counter()), |
| 984 | "another scope, same chore": QueueLocationPark(moveA, "ie", "", changed, counter()), |
| 985 | "another op": QueueLocationPark(moveB, "ie", "", loc, counter()), |
| 986 | "no id source": QueueLocationPark(moveA, "ie", "", loc, nil), |
| 987 | } { |
| 988 | before := mustJSON(t, c) |
| 989 | if err := fn(c); err == nil { |
| 990 | t.Errorf("%s: accepted", name) |
| 991 | } |
| 992 | if string(mustJSON(t, c)) != string(before) { |
| 993 | t.Errorf("%s: changed the claim", name) |
| 994 | } |
| 995 | } |
| 996 | } |
| 997 | |
| 998 | func TestResidueParksArePerGeneration(t *testing.T) { |
| 999 | c, scope := parkFixture(t) |
| 1000 | loc := scope("instances/" + c.Host + "/" + c.Holder.Incarnation) |
| 1001 | n := counter() |
| 1002 | gens := []string{"gen-4", RootGeneration, "gen-9"} |
| 1003 | if err := QueueResidueParks(moveA, "uk", loc, gens, n)(c); err != nil { |
| 1004 | t.Fatal(err) |
| 1005 | } |
| 1006 | if len(c.Obligations) != 3 { |
| 1007 | t.Fatalf("obligations: %+v", c.Obligations) |
| 1008 | } |
| 1009 | seen := map[string]bool{} |
| 1010 | for _, o := range c.Obligations { |
| 1011 | if o.Kind != "park-generation" || o.Op != moveA || o.Region != "uk" || o.Owner != "region:uk" || o.Park == nil || !sameParkScope(*o.Park, loc) || o.Retirement == "" || o.Box != "" { |
| 1012 | t.Fatalf("residue park: %+v", o) |
| 1013 | } |
| 1014 | seen[o.Generation] = true |
| 1015 | } |
| 1016 | if !seen["gen-4"] || !seen["gen-9"] || !seen["@root"] { |
| 1017 | t.Fatalf("generations: %v", seen) |
| 1018 | } |
| 1019 | c.Gen++ |
| 1020 | if err := c.Validate(); err != nil { |
| 1021 | t.Fatal(err) |
| 1022 | } |
| 1023 | // Each chore owns its scope: editing one does not touch another. |
| 1024 | c.Obligations[0].Park.RequestLimit = time.Hour |
| 1025 | if c.Obligations[1].Park.RequestLimit == time.Hour { |
| 1026 | t.Fatal("chores share one scope") |
| 1027 | } |
| 1028 | // A double call, a repeated generation in the list and a superset add |
| 1029 | // only what is missing, and spend ids only for that. |
| 1030 | ids := 0 |
| 1031 | next := func() string { ids++; return fmt.Sprintf("n%d", ids) } |
| 1032 | loc.RequestLimit = 2 * time.Minute |
| 1033 | c.Obligations[0].Park.RequestLimit = 2 * time.Minute |
| 1034 | if err := QueueResidueParks(moveA, "uk", loc, append(gens, "gen-4", "gen-11", "gen-11"), next)(c); err != nil { |
| 1035 | t.Fatal(err) |
| 1036 | } |
| 1037 | if len(c.Obligations) != 4 || ids != 2 || c.Obligations[3].Generation != "gen-11" { |
| 1038 | t.Fatalf("superset: %d obligations, %d ids", len(c.Obligations), ids) |
| 1039 | } |
| 1040 | // No generations is nothing to do; an empty name is malformed. |
| 1041 | if err := QueueResidueParks(moveA, "uk", loc, nil, next)(c); err != nil || len(c.Obligations) != 4 { |
| 1042 | t.Fatalf("none: %v", err) |
| 1043 | } |
| 1044 | if err := QueueResidueParks(moveA, "uk", loc, []string{"gen-12", ""}, next)(c); !errors.Is(err, ErrMalformed) { |
| 1045 | t.Fatalf("empty generation: %v", err) |
| 1046 | } |
| 1047 | // The staging lane is not a residue location. |
| 1048 | if err := QueueResidueParks(moveA, "uk", scope("moves/"+c.Host+"/"+moveA), []string{"gen-1"}, next)(c); err == nil { |
| 1049 | t.Fatal("residue parked a lane") |
| 1050 | } |
| 1051 | } |
| 1052 | |
| 1053 | func TestLocalParkAndTheEarlierStampsPaths(t *testing.T) { |
| 1054 | c, _ := parkFixture(t) |
| 1055 | c.Inventory = map[string]InventoryEntry{"uk-box": {Layout: "nested", Paths: map[string]uint64{"/data/a": 4, "/data/b": 7}}} |
| 1056 | if !InventoryHasStamp(c, "uk-box", 4) || InventoryHasStamp(c, "uk-box", 5) || InventoryHasStamp(c, "ie-box", 4) || InventoryHasStamp(c, "uk-box", 0) { |
| 1057 | t.Fatal("InventoryHasStamp") |
| 1058 | } |
| 1059 | ids := 0 |
| 1060 | newID := func() string { ids++; return fmt.Sprintf("l%d", ids) } |
| 1061 | if err := QueueLocalPark(moveA, "uk-box", newID)(c); err != nil { |
| 1062 | t.Fatal(err) |
| 1063 | } |
| 1064 | if err := QueueLocalPark(moveA, "uk-box", newID)(c); err != nil || ids != 2 || len(c.Obligations) != 1 { |
| 1065 | t.Fatalf("double call: %v ids=%d obligations=%d", err, ids, len(c.Obligations)) |
| 1066 | } |
| 1067 | o := c.Obligations[0] |
| 1068 | if o.Kind != "park-local" || o.Box != "uk-box" || o.Owner != "box:uk-box" || o.Op != moveA || o.Region != "" || o.Park != nil { |
| 1069 | t.Fatalf("local park: %+v", o) |
| 1070 | } |
| 1071 | c.Gen++ |
| 1072 | if err := c.Validate(); err != nil { |
| 1073 | t.Fatal(err) |
| 1074 | } |
| 1075 | // The replay with S' paths still there must not complete the chore. |
| 1076 | if err := CompleteLocalPark(moveA, "uk-box", 4)(c); !errors.Is(err, ErrWrongState) || len(c.Obligations) != 1 { |
| 1077 | t.Fatalf("complete with paths left: %v", err) |
| 1078 | } |
| 1079 | // With no S' paths the chore is completed under the op; a second time is a no-op. |
| 1080 | c.Inventory["uk-box"] = InventoryEntry{Layout: "nested", Paths: map[string]uint64{"/data/b": 7}} |
| 1081 | if err := CompleteLocalPark(moveA, "uk-box", 4)(c); err != nil || len(c.Obligations) != 0 { |
| 1082 | t.Fatalf("complete: %v %+v", err, c.Obligations) |
| 1083 | } |
| 1084 | if err := CompleteLocalPark(moveA, "uk-box", 4)(c); err != nil { |
| 1085 | t.Fatalf("replay: %v", err) |
| 1086 | } |
| 1087 | // It is the op's own chore only. |
| 1088 | if err := QueueLocalPark(moveA, "uk-box", newID)(c); err != nil { |
| 1089 | t.Fatal(err) |
| 1090 | } |
| 1091 | if err := CompleteLocalPark(moveB, "uk-box", 4)(c); !errors.Is(err, ErrStaleOp) || len(c.Obligations) != 1 { |
| 1092 | t.Fatalf("another op: %v", err) |
| 1093 | } |
| 1094 | if err := QueueLocalPark(moveA, "", newID)(c); err == nil { |
| 1095 | t.Fatal("a local park with no box") |
| 1096 | } |
| 1097 | } |
| 1098 | |
| 1099 | // The park queue helpers own only the obligations they append. Every |
| 1100 | // other claim field, and every other op's chore, is carried through. |
| 1101 | func TestParkQueueHelpersPreserveTheRestOfTheClaim(t *testing.T) { |
| 1102 | now := time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC) |
| 1103 | build := func() *Claim { |
| 1104 | c := movingTestClaim(t) |
| 1105 | c.Holder.Generation = "gen-7" |
| 1106 | c.Holder.Session = 9 |
| 1107 | c.Holder.WriterBox = "ie-box" |
| 1108 | c.Holder.Admissions = []AdmissionRecord{{Region: "ie", Session: 9, Box: "ie-box", Generation: "gen-7", Mode: "admitted", Coverage: "covered", Covered: true, |
| 1109 | WritesLost: &WritesLost{Box: "ie-box", Generation: "gen-6", Session: 8, RemoteMaxTXID: 41}, |
| 1110 | AdditionalWritesLost: []WritesLost{{Box: "ie-box", Generation: "gen-5", Session: 7, RemoteMaxTXID: 12}}}} |
| 1111 | c.Obligations = []Obligation{{ID: "o1", Op: "old", Kind: "park-replica", Region: "se", Retirement: "r", Owner: "region:se"}} |
| 1112 | c.Lost = []string{"se"} |
| 1113 | c.RetiredIncarnations = []string{"abc"} |
| 1114 | c.QuietAfter = map[string]time.Time{"se": now} |
| 1115 | c.Touched = Touched{Complete: true, Regions: []string{"ie", "uk"}, Boxes: []string{"ie-box", "uk-box"}} |
| 1116 | c.Inventory = map[string]InventoryEntry{"ie-box": {Layout: "nested", Paths: map[string]uint64{"/data/x": 3}}} |
| 1117 | if err := BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", now)(c); err != nil { |
| 1118 | t.Fatal(err) |
| 1119 | } |
| 1120 | inState(c, KindLanding, "committed") |
| 1121 | if err := c.Validate(); err != nil { |
| 1122 | t.Fatalf("fixture: %v", err) |
| 1123 | } |
| 1124 | return c |
| 1125 | } |
| 1126 | base := build() |
| 1127 | scope := func(loc string) ParkScope { |
| 1128 | return ParkScope{Location: loc, Incarnation: base.Holder.Incarnation, EvidenceAt: now, RequestLimit: time.Minute} |
| 1129 | } |
| 1130 | loc := scope(base.Holder.Location) |
| 1131 | lane := scope("moves/" + base.Host + "/" + moveA) |
| 1132 | cases := map[string]struct { |
| 1133 | fn func(*Claim) error |
| 1134 | want int |
| 1135 | }{ |
| 1136 | "location+box": {QueueLocationPark(moveA, "ie", "ie-box", loc, counter()), 2}, |
| 1137 | "staging": {QueueStagingPark(moveA, "uk", lane, counter()), 1}, |
| 1138 | "local": {QueueLocalPark(moveA, "uk-box", counter()), 1}, |
| 1139 | "residue": {QueueResidueParks(moveA, "uk", loc, []string{"gen-1", RootGeneration}, counter()), 2}, |
| 1140 | } |
| 1141 | for name, tc := range cases { |
| 1142 | t.Run(name, func(t *testing.T) { |
| 1143 | c := build() |
| 1144 | before := mustJSON(t, c) |
| 1145 | if err := tc.fn(c); err != nil { |
| 1146 | t.Fatal(err) |
| 1147 | } |
| 1148 | if len(c.Obligations) != 1+tc.want || !reflect.DeepEqual(c.Obligations[0], build().Obligations[0]) { |
| 1149 | t.Fatalf("obligations: %+v", c.Obligations) |
| 1150 | } |
| 1151 | var round Claim |
| 1152 | if err := json.Unmarshal(before, &round); err != nil { |
| 1153 | t.Fatal(err) |
| 1154 | } |
| 1155 | round.Obligations = c.Obligations |
| 1156 | if !reflect.DeepEqual(round, *c) { |
| 1157 | t.Fatalf("changed more than the obligations:\nbefore %s\nafter %s", before, mustJSON(t, c)) |
| 1158 | } |
| 1159 | c.Gen++ |
| 1160 | if err := c.Validate(); err != nil { |
| 1161 | t.Fatal(err) |
| 1162 | } |
| 1163 | }) |
| 1164 | } |
| 1165 | } |
| 1166 | |
| 1167 | // probedLanding drives the claim to landing/probed the way the landing does: |
| 1168 | // the flip, then every record phase mirrored through SetMovePhase. The |
| 1169 | // target's probe admission is on the holder, its predecessor in region ie. |
| 1170 | func probedLanding(t *testing.T) *Claim { |
| 1171 | t.Helper() |
| 1172 | c := movingToSealed(t) |
| 1173 | c.Gen = 7 |
| 1174 | if err := Flip(moveA, "uk-box", 3, "gen-7")(c); err != nil { |
| 1175 | t.Fatal(err) |
| 1176 | } |
| 1177 | for _, step := range [][2]string{{"checking", "parking"}, {"parking", "building"}, {"building", "published"}, {"published", "local"}, {"local", "configured"}, {"configured", "restored"}, {"restored", "probing"}, {"probing", "probed"}} { |
| 1178 | if err := SetMovePhase(moveA, KindLanding, step[0], step[1])(c); err != nil { |
| 1179 | t.Fatal(err) |
| 1180 | } |
| 1181 | } |
| 1182 | c.Holder.Session = 9 |
| 1183 | c.Holder.Admissions = []AdmissionRecord{{Region: "uk", PredecessorRegion: "ie", Session: 9, Box: "uk-box", Generation: moveA, Predecessor: "gen-7", Mode: "cold", Coverage: "sealed"}} |
| 1184 | return c |
| 1185 | } |
| 1186 | |
| 1187 | func commitScopes(c *Claim) (src, stg ParkScope) { |
| 1188 | at := time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC) |
| 1189 | src = ParkScope{Location: c.Holder.Location, Incarnation: c.Holder.Incarnation, EvidenceAt: at, RequestLimit: 120 * time.Second} |
| 1190 | stg = ParkScope{Location: "moves/" + c.Host + "/" + moveA, Incarnation: c.Holder.Incarnation, EvidenceAt: at, RequestLimit: 120 * time.Second} |
| 1191 | return src, stg |
| 1192 | } |
| 1193 | |
| 1194 | func obligationKinds(c *Claim) map[string]int { |
| 1195 | kinds := map[string]int{} |
| 1196 | for _, o := range c.Obligations { |
| 1197 | kinds[o.Kind+"/"+o.Region+o.Box]++ |
| 1198 | } |
| 1199 | return kinds |
| 1200 | } |
| 1201 | |
| 1202 | // wCommitCoverage, wCommitReplayed, H3, H5(b). |
| 1203 | func TestCommitQueuesObligationsAndCoversTheSourceRegion(t *testing.T) { |
| 1204 | for _, from := range []string{"probed"} { |
| 1205 | c := probedLanding(t) |
| 1206 | c.Op.Phase = from |
| 1207 | // The source box holds paths of the stay the move leaves: stamp S = 3. |
| 1208 | c.Inventory = map[string]InventoryEntry{"ie-box": {Layout: "flat", Paths: map[string]uint64{"/data/a": 3}}} |
| 1209 | stamp, session, gen := c.Holder.Stamp, c.Holder.Session, c.Holder.Generation |
| 1210 | ids := 0 |
| 1211 | newID := func() string { ids++; return fmt.Sprintf("id%d", ids) } |
| 1212 | src, stg := commitScopes(c) |
| 1213 | if err := Commit(moveA, "ie", "ie-box", "uk", src, stg, newID)(c); err != nil { |
| 1214 | t.Fatalf("from %s: %v", from, err) |
| 1215 | } |
| 1216 | if c.Op.Kind != KindLanding || c.Op.Phase != PhaseCommitted || c.Op.Exec != "uk-box" { |
| 1217 | t.Fatalf("after commit: %+v", c.Op) |
| 1218 | } |
| 1219 | // H3: the holder's stamp, session and generation are the flip's, untouched. |
| 1220 | if c.Holder.Stamp != stamp || c.Holder.Session != session || c.Holder.Generation != gen { |
| 1221 | t.Fatalf("commit touched the holder: %+v", c.Holder) |
| 1222 | } |
| 1223 | k := obligationKinds(c) |
| 1224 | if k["park-replica/ie"] != 1 || k["park-local/ie-box"] != 1 || k["park-replica/uk"] != 1 || len(c.Obligations) != 3 { |
| 1225 | t.Fatalf("obligations: %+v", c.Obligations) |
| 1226 | } |
| 1227 | for _, o := range c.Obligations { |
| 1228 | if o.Kind == "park-replica" && o.Region == "ie" && (o.Park == nil || o.Park.Location != c.Holder.Location || o.Owner != "region:ie") { |
| 1229 | t.Fatalf("source park: %+v", o) |
| 1230 | } |
| 1231 | if o.Kind == "park-replica" && o.Region == "uk" && (o.Park == nil || o.Park.Location != stg.Location) { |
| 1232 | t.Fatalf("staging park: %+v", o) |
| 1233 | } |
| 1234 | } |
| 1235 | if !c.Holder.Admissions[0].Covered { |
| 1236 | t.Fatal("the source-region predecessor is not covered (wCommitCoverage)") |
| 1237 | } |
| 1238 | if err := c.Validate(); err != nil { |
| 1239 | t.Fatalf("the committed claim must validate: %v", err) |
| 1240 | } |
| 1241 | // Replay: nothing doubles and no ids are minted (wCommitReplayed). |
| 1242 | before, usedIDs := mustJSON(t, c), ids |
| 1243 | if err := Commit(moveA, "ie", "ie-box", "uk", src, stg, newID)(c); err != nil || ids != usedIDs || string(mustJSON(t, c)) != string(before) { |
| 1244 | t.Fatalf("replay: %v ids %d->%d", err, usedIDs, ids) |
| 1245 | } |
| 1246 | } |
| 1247 | } |
| 1248 | |
| 1249 | // H5(b): the source's park-local is queued only when the source box lists |
| 1250 | // paths of the journalled source stamp S. |
| 1251 | func TestCommitQueuesTheSourceParkLocalOnlyForTheSourceStamp(t *testing.T) { |
| 1252 | for name, inv := range map[string]map[string]InventoryEntry{ |
| 1253 | "no inventory": nil, |
| 1254 | "other stamp only": {"ie-box": {Layout: "flat", Paths: map[string]uint64{"/data/a": 2}}}, |
| 1255 | "stamp on another box": {"uk-box": {Layout: "flat", Paths: map[string]uint64{"/data/a": 3}}}, |
| 1256 | } { |
| 1257 | c := probedLanding(t) |
| 1258 | c.Inventory = inv |
| 1259 | ids := 0 |
| 1260 | newID := func() string { ids++; return fmt.Sprintf("id%d", ids) } |
| 1261 | src, stg := commitScopes(c) |
| 1262 | if err := Commit(moveA, "ie", "ie-box", "uk", src, stg, newID)(c); err != nil { |
| 1263 | t.Fatalf("%s: %v", name, err) |
| 1264 | } |
| 1265 | k := obligationKinds(c) |
| 1266 | if k["park-replica/ie"] != 1 || k["park-replica/uk"] != 1 || len(c.Obligations) != 2 { |
| 1267 | t.Fatalf("%s: obligations %+v", name, c.Obligations) |
| 1268 | } |
| 1269 | if !c.Holder.Admissions[0].Covered { |
| 1270 | t.Fatalf("%s: the source region is not covered", name) |
| 1271 | } |
| 1272 | } |
| 1273 | } |
| 1274 | |
| 1275 | // wCommitInterrupted: a commit that is refused leaves the claim as it was. |
| 1276 | func TestCommitRefusesAnythingButAProbingTargetLanding(t *testing.T) { |
| 1277 | newID := func() string { return "x" } |
| 1278 | type commitFn = func(c *Claim, src, stg ParkScope) func(*Claim) error |
| 1279 | std := func(op, region, box string) commitFn { |
| 1280 | return func(c *Claim, src, stg ParkScope) func(*Claim) error { |
| 1281 | return Commit(op, region, box, "uk", src, stg, newID) |
| 1282 | } |
| 1283 | } |
| 1284 | for name, tc := range map[string]struct { |
| 1285 | mutate func(c *Claim, src, stg *ParkScope) |
| 1286 | fn commitFn |
| 1287 | want error |
| 1288 | }{ |
| 1289 | "serving claim": {func(c *Claim, _, _ *ParkScope) { c.Op = Op{Kind: KindServing, Exec: "ie-box"} }, std(moveA, "ie", "ie-box"), ErrStaleOp}, |
| 1290 | "another op": {func(*Claim, *ParkScope, *ParkScope) {}, std(moveB, "ie", "ie-box"), ErrStaleOp}, |
| 1291 | "moving": {func(c *Claim, _, _ *ParkScope) { c.Op.Kind, c.Op.Phase, c.Op.Exec = KindMoving, PhaseSealed, "" }, std(moveA, "ie", "ie-box"), ErrWrongState}, |
| 1292 | "landing before probing": {func(c *Claim, _, _ *ParkScope) { c.Op.Phase = "restored" }, std(moveA, "ie", "ie-box"), ErrWrongState}, |
| 1293 | "exec not the target": {func(c *Claim, _, _ *ParkScope) { c.Op.Exec = "ie-box" }, std(moveA, "ie", "ie-box"), ErrWrongState}, |
| 1294 | "exec none": {func(c *Claim, _, _ *ParkScope) { c.Op.Exec = "" }, std(moveA, "ie", "ie-box"), ErrWrongState}, |
| 1295 | "source box is not the source": {func(*Claim, *ParkScope, *ParkScope) {}, std(moveA, "ie", "uk-box"), ErrWrongState}, |
| 1296 | "source region is the target": {func(*Claim, *ParkScope, *ParkScope) {}, std(moveA, "uk", "ie-box"), ErrWrongState}, |
| 1297 | "source park is not the holder's location": {func(_ *Claim, s, _ *ParkScope) { s.Location = "instances/a.go.example.com/other" }, std(moveA, "ie", "ie-box"), ErrMalformed}, |
| 1298 | "staging park is another lane": {func(c *Claim, _, g *ParkScope) { g.Location = "moves/" + c.Host + "/" + moveB }, std(moveA, "ie", "ie-box"), ErrMalformed}, |
| 1299 | } { |
| 1300 | c := probedLanding(t) |
| 1301 | c.Inventory = map[string]InventoryEntry{"ie-box": {Layout: "flat", Paths: map[string]uint64{"/data/a": 3}}} |
| 1302 | src, stg := commitScopes(c) |
| 1303 | tc.mutate(c, &src, &stg) |
| 1304 | before := mustJSON(t, c) |
| 1305 | if err := tc.fn(c, src, stg)(c); !errors.Is(err, tc.want) { |
| 1306 | t.Fatalf("%s: got %v, want %v", name, err, tc.want) |
| 1307 | } |
| 1308 | if after := mustJSON(t, c); string(after) != string(before) { |
| 1309 | t.Fatalf("%s mutated the claim:\nbefore %s\nafter %s", name, before, after) |
| 1310 | } |
| 1311 | } |
| 1312 | } |
| 1313 | |
| 1314 | // ---- Plan 4 Task 2: the rollback and the return (§7.8, K4, K8) ---- |
| 1315 | |
| 1316 | const ( |
| 1317 | rbA = "rb_abcdefghijklmnopqrstuvwxyz" |
| 1318 | rbB = "rb_bcdefghijklmnopqrstuvwxyza" |
| 1319 | stoppedTarget = "moves/bab/app/h/rb/stopped-target.json" |
| 1320 | ) |
| 1321 | |
| 1322 | // targetWroteLanding is probedLanding with the target's probe admitted as |
| 1323 | // the writer: holder.generation and writer_box are the target's, and the |
| 1324 | // last admission record is the target's writer (its predecessor the |
| 1325 | // source's generation in ie). The flip at gen 7 made the target's |
| 1326 | // installation stamp F = 8. |
| 1327 | func targetWroteLanding(t *testing.T) *Claim { |
| 1328 | t.Helper() |
| 1329 | c := probedLanding(t) |
| 1330 | c.Holder.Generation, c.Holder.WriterBox = moveA, "uk-box" |
| 1331 | if c.Holder.Stamp != 8 { |
| 1332 | t.Fatalf("fixture: the flip's stamp is %d, want 8", c.Holder.Stamp) |
| 1333 | } |
| 1334 | return c |
| 1335 | } |
| 1336 | |
| 1337 | // rolledBack is targetWroteLanding after the rollback's claim half. |
| 1338 | func rolledBack(t *testing.T) *Claim { |
| 1339 | t.Helper() |
| 1340 | c := targetWroteLanding(t) |
| 1341 | if err := RollBack(moveA, rbA)(c); err != nil { |
| 1342 | t.Fatal(err) |
| 1343 | } |
| 1344 | return c |
| 1345 | } |
| 1346 | |
| 1347 | func releaseScope(c *Claim) ParkScope { |
| 1348 | return ParkScope{Location: c.Holder.Location, Incarnation: c.Holder.Incarnation, EvidenceAt: time.Date(2026, 10, 6, 11, 0, 0, 0, time.UTC), RequestLimit: 2 * time.Minute} |
| 1349 | } |
| 1350 | |
| 1351 | // released is rolledBack after the target's release, with no park-local |
| 1352 | // (the target listed no paths of F). |
| 1353 | func released(t *testing.T) *Claim { |
| 1354 | t.Helper() |
| 1355 | c := rolledBack(t) |
| 1356 | if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, releaseScope(c), counter())(c); err != nil { |
| 1357 | t.Fatal(err) |
| 1358 | } |
| 1359 | return c |
| 1360 | } |
| 1361 | |
| 1362 | func validAfterWrite(t *testing.T, c *Claim) { |
| 1363 | t.Helper() |
| 1364 | d := *c |
| 1365 | d.Gen++ |
| 1366 | if err := d.Validate(); err != nil { |
| 1367 | t.Fatalf("the claim after the write must validate: %v", err) |
| 1368 | } |
| 1369 | } |
| 1370 | |
| 1371 | // The commit and the rollback race on the claim as on the record: whichever |
| 1372 | // writes first, the other is refused and leaves the claim as it found it. |
| 1373 | func TestRollBackRacesTheCommitOnTheClaim(t *testing.T) { |
| 1374 | newID := counter() |
| 1375 | // The commit first. |
| 1376 | c := targetWroteLanding(t) |
| 1377 | src, stg := commitScopes(c) |
| 1378 | if err := Commit(moveA, "ie", "ie-box", "uk", src, stg, newID)(c); err != nil { |
| 1379 | t.Fatal(err) |
| 1380 | } |
| 1381 | before := mustJSON(t, c) |
| 1382 | if err := RollBack(moveA, rbA)(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) { |
| 1383 | t.Fatalf("rollback after the commit: %v", err) |
| 1384 | } |
| 1385 | // The rollback first. |
| 1386 | d := targetWroteLanding(t) |
| 1387 | if err := RollBack(moveA, rbA)(d); err != nil { |
| 1388 | t.Fatal(err) |
| 1389 | } |
| 1390 | if d.Op.Kind != KindRestoring || d.Op.Phase != PhaseReleasing || d.Op.Exec != "uk-box" || d.Op.Intent["rollback"] != rbA || |
| 1391 | d.Op.ID != moveA || len(d.Op.Participants) != 2 { |
| 1392 | t.Fatalf("op after the rollback: %+v", d.Op) |
| 1393 | } |
| 1394 | // J15: the target installation's stamp F is journalled for its release. |
| 1395 | if d.Op.Journal[JournalTargetStamp] != "8" { |
| 1396 | t.Fatalf("target stamp journal %q", d.Op.Journal[JournalTargetStamp]) |
| 1397 | } |
| 1398 | if d.Holder.Stamp != 8 || d.Holder.Generation != moveA { |
| 1399 | t.Fatalf("the rollback touched the holder: %+v", d.Holder) |
| 1400 | } |
| 1401 | validAfterWrite(t, d) |
| 1402 | before = mustJSON(t, d) |
| 1403 | if err := Commit(moveA, "ie", "ie-box", "uk", src, stg, newID)(d); !errors.Is(err, ErrWrongState) || string(mustJSON(t, d)) != string(before) { |
| 1404 | t.Fatalf("commit after the rollback: %v", err) |
| 1405 | } |
| 1406 | // A replay of the same rollback is a no-op; another rollback id is refused. |
| 1407 | if err := RollBack(moveA, rbA)(d); err != nil || string(mustJSON(t, d)) != string(before) { |
| 1408 | t.Fatalf("replay: %v", err) |
| 1409 | } |
| 1410 | if err := RollBack(moveA, rbB)(d); !errors.Is(err, ErrWrongState) || string(mustJSON(t, d)) != string(before) { |
| 1411 | t.Fatalf("another rollback id: %v", err) |
| 1412 | } |
| 1413 | // Every landing phase before the commit rolls back. |
| 1414 | for _, phase := range []string{"checking", "parking", "building", "published", "local", "configured", "restored", "probing", "probed"} { |
| 1415 | e := targetWroteLanding(t) |
| 1416 | e.Op.Phase = phase |
| 1417 | if err := RollBack(moveA, rbA)(e); err != nil || e.Op.Kind != KindRestoring || e.Op.Phase != PhaseReleasing { |
| 1418 | t.Fatalf("from landing/%s: %v %+v", phase, err, e.Op) |
| 1419 | } |
| 1420 | } |
| 1421 | // Refusals leave the claim alone. |
| 1422 | for name, tc := range map[string]struct { |
| 1423 | mutate func(*Claim) |
| 1424 | op, rb string |
| 1425 | want error |
| 1426 | }{ |
| 1427 | "another op": {func(*Claim) {}, moveB, rbA, ErrStaleOp}, |
| 1428 | "a move id": {func(*Claim) {}, moveA, moveB, ErrMalformed}, |
| 1429 | "no id": {func(*Claim) {}, moveA, "", ErrMalformed}, |
| 1430 | "short id": {func(*Claim) {}, moveA, "rb_abc", ErrMalformed}, |
| 1431 | "serving": {func(c *Claim) { c.Op = Op{Kind: KindServing, Exec: "uk-box"} }, moveA, rbA, ErrStaleOp}, |
| 1432 | "moving, sealed": {func(c *Claim) { c.Op.Kind, c.Op.Phase, c.Op.Exec = KindMoving, PhaseSealed, "" }, moveA, rbA, ErrWrongState}, |
| 1433 | "reverting": {func(c *Claim) { c.Op.Kind, c.Op.Phase, c.Op.Exec = KindReverting, "", "ie-box" }, moveA, rbA, ErrWrongState}, |
| 1434 | "landing, no holder": {func(c *Claim) { c.Holder = nil }, moveA, rbA, ErrWrongState}, |
| 1435 | "landing, committed": {func(c *Claim) { c.Op.Phase = PhaseCommitted }, moveA, rbA, ErrWrongState}, |
| 1436 | "restoring under rbB": {func(c *Claim) { c.Op.Kind, c.Op.Phase, c.Op.Intent["rollback"] = KindRestoring, PhaseReleasing, rbB }, moveA, rbA, ErrWrongState}, |
| 1437 | } { |
| 1438 | e := targetWroteLanding(t) |
| 1439 | tc.mutate(e) |
| 1440 | before := mustJSON(t, e) |
| 1441 | if err := RollBack(tc.op, tc.rb)(e); !errors.Is(err, tc.want) || string(mustJSON(t, e)) != string(before) { |
| 1442 | t.Fatalf("%s: got %v, want %v", name, err, tc.want) |
| 1443 | } |
| 1444 | } |
| 1445 | } |
| 1446 | |
| 1447 | // J16: a record already at landing whose claim is still moving/sealed (the |
| 1448 | // flip's claim half not yet written) rolls back through RollBackFlipped, |
| 1449 | // which writes the flip first, in the same write. |
| 1450 | func TestRollBackFlippedWritesTheFlipFirst(t *testing.T) { |
| 1451 | c := movingToSealed(t) |
| 1452 | c.Gen = 7 |
| 1453 | before := mustJSON(t, c) |
| 1454 | if err := RollBack(moveA, rbA)(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) { |
| 1455 | t.Fatalf("plain rollback over moving/sealed: %v", err) |
| 1456 | } |
| 1457 | // The flip half refuses (the source's stamp is wrong): nothing is written. |
| 1458 | if err := RollBackFlipped(moveA, rbA, "uk-box", 2, "gen-7")(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) { |
| 1459 | t.Fatalf("a refused flip half: %v\nbefore %s\nafter %s", err, before, mustJSON(t, c)) |
| 1460 | } |
| 1461 | if err := RollBackFlipped(moveA, "", "uk-box", 3, "gen-7")(c); !errors.Is(err, ErrMalformed) || string(mustJSON(t, c)) != string(before) { |
| 1462 | t.Fatalf("a malformed rollback id: %v", err) |
| 1463 | } |
| 1464 | if err := RollBackFlipped(moveA, rbA, "uk-box", 3, "gen-7")(c); err != nil { |
| 1465 | t.Fatal(err) |
| 1466 | } |
| 1467 | if c.Op.Kind != KindRestoring || c.Op.Phase != PhaseReleasing || c.Op.Exec != "uk-box" || c.Op.Intent["rollback"] != rbA { |
| 1468 | t.Fatalf("op: %+v", c.Op) |
| 1469 | } |
| 1470 | if c.Holder.Stamp != 8 || c.Op.Journal[JournalTargetStamp] != "8" || c.Op.Journal[JournalSourceStamp] != "3" || c.Op.Journal[JournalSrcGen] != "gen-7" { |
| 1471 | t.Fatalf("stamps: holder %d journal %v", c.Holder.Stamp, c.Op.Journal) |
| 1472 | } |
| 1473 | validAfterWrite(t, c) |
| 1474 | // Over an already flipped (landing) claim it is RollBack, and a replay is a no-op. |
| 1475 | d := targetWroteLanding(t) |
| 1476 | if err := RollBackFlipped(moveA, rbA, "uk-box", 3, "gen-7")(d); err != nil || d.Op.Kind != KindRestoring { |
| 1477 | t.Fatalf("over landing: %v %+v", err, d.Op) |
| 1478 | } |
| 1479 | after := mustJSON(t, d) |
| 1480 | if err := RollBackFlipped(moveA, rbA, "uk-box", 3, "gen-7")(d); err != nil || string(mustJSON(t, d)) != string(after) { |
| 1481 | t.Fatalf("replay: %v", err) |
| 1482 | } |
| 1483 | } |
| 1484 | |
| 1485 | // §7.8 released, claim rev 6.14 item 6 (R12-3), J15. |
| 1486 | func TestTargetReleasedCoversTheTargetRegion(t *testing.T) { |
| 1487 | build := func(inventory map[string]InventoryEntry) *Claim { |
| 1488 | c := targetWroteLanding(t) |
| 1489 | // A restarted probe: its record's predecessor is the dead probe's |
| 1490 | // generation, in the target's region. |
| 1491 | c.Holder.Admissions = append(c.Holder.Admissions, AdmissionRecord{Region: "uk", PredecessorRegion: "uk", Session: 10, Box: "uk-box", Generation: "gen-r2", Predecessor: moveA, Mode: "cold"}) |
| 1492 | c.Holder.Session, c.Holder.Generation = 10, "gen-r2" |
| 1493 | c.Inventory = inventory |
| 1494 | if err := RollBack(moveA, rbA)(c); err != nil { |
| 1495 | t.Fatal(err) |
| 1496 | } |
| 1497 | return c |
| 1498 | } |
| 1499 | withF := map[string]InventoryEntry{"uk-box": {Layout: "nested", Paths: map[string]uint64{"/data/uk": 8}}} |
| 1500 | c := build(withF) |
| 1501 | ids := 0 |
| 1502 | newID := func() string { ids++; return fmt.Sprintf("t%d", ids) } |
| 1503 | holder := *c.Holder |
| 1504 | if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, releaseScope(c), newID)(c); err != nil { |
| 1505 | t.Fatal(err) |
| 1506 | } |
| 1507 | if c.Op.Kind != KindRestoring || c.Op.Phase != PhaseReleased || c.Op.Exec != "" || c.Op.Journal[JournalEvidenceTarget] != stoppedTarget { |
| 1508 | t.Fatalf("op: %+v", c.Op) |
| 1509 | } |
| 1510 | k := obligationKinds(c) |
| 1511 | if k["park-replica/uk"] != 1 || k["park-local/uk-box"] != 1 || len(c.Obligations) != 2 { |
| 1512 | t.Fatalf("obligations: %+v", c.Obligations) |
| 1513 | } |
| 1514 | for _, o := range c.Obligations { |
| 1515 | if o.Op != moveA || (o.Kind == "park-replica" && (o.Park == nil || o.Park.Location != c.Holder.Location || o.Owner != "region:uk")) { |
| 1516 | t.Fatalf("obligation: %+v", o) |
| 1517 | } |
| 1518 | } |
| 1519 | if c.Holder.Admissions[0].Covered || !c.Holder.Admissions[1].Covered { |
| 1520 | t.Fatalf("coverage: ie-predecessor %v, uk-predecessor %v", c.Holder.Admissions[0].Covered, c.Holder.Admissions[1].Covered) |
| 1521 | } |
| 1522 | if c.Holder.Stamp != holder.Stamp || c.Holder.Session != holder.Session || c.Holder.Generation != holder.Generation || c.Holder.WriterBox != holder.WriterBox { |
| 1523 | t.Fatalf("the release touched the holder: %+v", c.Holder) |
| 1524 | } |
| 1525 | validAfterWrite(t, c) |
| 1526 | // A replay queues nothing and mints no ids, even once a chore is done. |
| 1527 | used, before := ids, mustJSON(t, c) |
| 1528 | if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, releaseScope(c), newID)(c); err != nil || ids != used || string(mustJSON(t, c)) != string(before) { |
| 1529 | t.Fatalf("replay: %v ids %d->%d", err, used, ids) |
| 1530 | } |
| 1531 | c.Obligations = c.Obligations[:1] |
| 1532 | before = mustJSON(t, c) |
| 1533 | if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, releaseScope(c), newID)(c); err != nil || ids != used || string(mustJSON(t, c)) != string(before) { |
| 1534 | t.Fatalf("replay after a completed chore: %v ids %d->%d", err, used, ids) |
| 1535 | } |
| 1536 | // J15: no paths of F on the target, no park-local. |
| 1537 | for name, inv := range map[string]map[string]InventoryEntry{ |
| 1538 | "no inventory": nil, |
| 1539 | "another stamp": {"uk-box": {Layout: "nested", Paths: map[string]uint64{"/data/uk": 3}}}, |
| 1540 | "F on another box": {"ie-box": {Layout: "nested", Paths: map[string]uint64{"/data/ie": 8}}}, |
| 1541 | } { |
| 1542 | d := build(inv) |
| 1543 | if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, releaseScope(d), counter())(d); err != nil { |
| 1544 | t.Fatalf("%s: %v", name, err) |
| 1545 | } |
| 1546 | if k := obligationKinds(d); k["park-replica/uk"] != 1 || len(d.Obligations) != 1 { |
| 1547 | t.Fatalf("%s: obligations %+v", name, d.Obligations) |
| 1548 | } |
| 1549 | } |
| 1550 | // Refusals leave the claim as found. |
| 1551 | for name, tc := range map[string]struct { |
| 1552 | mutate func(*Claim) |
| 1553 | box, region, ev, op string |
| 1554 | want error |
| 1555 | }{ |
| 1556 | "another op": {func(*Claim) {}, "uk-box", "uk", stoppedTarget, moveB, ErrStaleOp}, |
| 1557 | "not rolled back": {func(c *Claim) { c.Op.Kind, c.Op.Phase = KindLanding, "probed" }, "uk-box", "uk", stoppedTarget, moveA, ErrWrongState}, |
| 1558 | "the source's box": {func(*Claim) {}, "ie-box", "uk", stoppedTarget, moveA, ErrWrongState}, |
| 1559 | "the source's region": {func(*Claim) {}, "uk-box", "ie", stoppedTarget, moveA, ErrWrongState}, |
| 1560 | "no evidence": {func(*Claim) {}, "uk-box", "uk", "", moveA, ErrMalformed}, |
| 1561 | "exec is not the target": {func(c *Claim) { c.Op.Exec = "ie-box" }, "uk-box", "uk", stoppedTarget, moveA, ErrWrongState}, |
| 1562 | "released, other evidence": {func(c *Claim) { |
| 1563 | c.Op.Phase, c.Op.Exec, c.Op.Journal[JournalEvidenceTarget] = PhaseReleased, "", "other" |
| 1564 | }, "uk-box", "uk", stoppedTarget, moveA, ErrWrongState}, |
| 1565 | } { |
| 1566 | d := build(withF) |
| 1567 | tc.mutate(d) |
| 1568 | before := mustJSON(t, d) |
| 1569 | if err := TargetReleased(tc.op, tc.box, tc.region, tc.ev, releaseScope(d), counter())(d); !errors.Is(err, tc.want) || string(mustJSON(t, d)) != string(before) { |
| 1570 | t.Fatalf("%s: got %v, want %v", name, err, tc.want) |
| 1571 | } |
| 1572 | } |
| 1573 | // A location park that is not the holder's location refuses the whole |
| 1574 | // write: exec, phase and coverage stay as found. |
| 1575 | d := build(withF) |
| 1576 | before = mustJSON(t, d) |
| 1577 | bad := releaseScope(d) |
| 1578 | bad.Location = "instances/" + d.Host |
| 1579 | if err := TargetReleased(moveA, "uk-box", "uk", stoppedTarget, bad, counter())(d); !errors.Is(err, ErrMalformed) || string(mustJSON(t, d)) != string(before) { |
| 1580 | t.Fatalf("bad scope: %v\nbefore %s\nafter %s", err, before, mustJSON(t, d)) |
| 1581 | } |
| 1582 | } |
| 1583 | |
| 1584 | // Review Focus 3 (wCleanReturnIdentity, INV-16, R11-4): the clean return |
| 1585 | // names the source's generation and stamp again and leaves the admissions, |
| 1586 | // whose last record is the target's writer, for the re-admission to name. |
| 1587 | func TestReturnToSourceCleanNamesSrcGenAndKeepsAdmissions(t *testing.T) { |
| 1588 | c := released(t) |
| 1589 | admissions := append([]AdmissionRecord(nil), c.Holder.Admissions...) |
| 1590 | session, writer := c.Holder.Session, c.Holder.WriterBox |
| 1591 | if err := ReturnToSource(moveA, "ie-box", true, ReturnFence{})(c); err != nil { |
| 1592 | t.Fatal(err) |
| 1593 | } |
| 1594 | if c.Op.Kind != KindRestoring || c.Op.Phase != PhaseClaimed || c.Op.Exec != "ie-box" { |
| 1595 | t.Fatalf("op: %+v", c.Op) |
| 1596 | } |
| 1597 | if c.Holder.Generation != "gen-7" || c.Holder.Stamp != 3 { |
| 1598 | t.Fatalf("holder generation %q stamp %d, want gen-7 and 3", c.Holder.Generation, c.Holder.Stamp) |
| 1599 | } |
| 1600 | if !reflect.DeepEqual(c.Holder.Admissions, admissions) || c.Holder.Session != session || c.Holder.WriterBox != writer { |
| 1601 | t.Fatalf("the return touched the admissions: %+v", c.Holder) |
| 1602 | } |
| 1603 | if last := c.Holder.Admissions[len(c.Holder.Admissions)-1]; last.Box != "uk-box" || last.Generation != moveA { |
| 1604 | t.Fatalf("the last admission is not the target's writer: %+v", last) |
| 1605 | } |
| 1606 | validAfterWrite(t, c) |
| 1607 | // A replay is a no-op. |
| 1608 | before := mustJSON(t, c) |
| 1609 | if err := ReturnToSource(moveA, "ie-box", true, ReturnFence{})(c); err != nil || string(mustJSON(t, c)) != string(before) { |
| 1610 | t.Fatalf("replay: %v", err) |
| 1611 | } |
| 1612 | // Refusals. |
| 1613 | for name, tc := range map[string]struct { |
| 1614 | claim func() *Claim |
| 1615 | op string |
| 1616 | source string |
| 1617 | want error |
| 1618 | }{ |
| 1619 | "another op": {func() *Claim { return released(t) }, moveB, "ie-box", ErrStaleOp}, |
| 1620 | "the target": {func() *Claim { return released(t) }, moveA, "uk-box", ErrWrongState}, |
| 1621 | "no source": {func() *Claim { return released(t) }, moveA, "", ErrMalformed}, |
| 1622 | "before the release": {func() *Claim { return rolledBack(t) }, moveA, "ie-box", ErrWrongState}, |
| 1623 | "landing": {func() *Claim { return targetWroteLanding(t) }, moveA, "ie-box", ErrWrongState}, |
| 1624 | "no evidence": {func() *Claim { |
| 1625 | c := released(t) |
| 1626 | delete(c.Op.Journal, JournalEvidenceTarget) |
| 1627 | return c |
| 1628 | }, moveA, "ie-box", ErrWrongState}, |
| 1629 | "no source stamp": {func() *Claim { |
| 1630 | c := released(t) |
| 1631 | delete(c.Op.Journal, JournalSourceStamp) |
| 1632 | return c |
| 1633 | }, moveA, "ie-box", ErrWrongState}, |
| 1634 | "no source gen": {func() *Claim { |
| 1635 | c := released(t) |
| 1636 | delete(c.Op.Journal, JournalSrcGen) |
| 1637 | return c |
| 1638 | }, moveA, "ie-box", ErrWrongState}, |
| 1639 | "exec still a box": {func() *Claim { |
| 1640 | c := released(t) |
| 1641 | c.Op.Exec = "uk-box" |
| 1642 | return c |
| 1643 | }, moveA, "ie-box", ErrWrongState}, |
| 1644 | "claimed by the other": {func() *Claim { |
| 1645 | c := released(t) |
| 1646 | c.Op.Phase, c.Op.Exec = PhaseClaimed, "uk-box" |
| 1647 | return c |
| 1648 | }, moveA, "ie-box", ErrWrongState}, |
| 1649 | "probing": {func() *Claim { |
| 1650 | c := released(t) |
| 1651 | c.Op.Phase, c.Op.Exec = PhaseProbing, "ie-box" |
| 1652 | return c |
| 1653 | }, moveA, "ie-box", ErrWrongState}, |
| 1654 | } { |
| 1655 | c := tc.claim() |
| 1656 | before := mustJSON(t, c) |
| 1657 | if err := ReturnToSource(tc.op, tc.source, true, ReturnFence{})(c); !errors.Is(err, tc.want) || string(mustJSON(t, c)) != string(before) { |
| 1658 | t.Fatalf("%s: got %v, want %v", name, err, tc.want) |
| 1659 | } |
| 1660 | } |
| 1661 | } |
| 1662 | |
| 1663 | // A dirty return names the source's stamp again and leaves holder.generation |
| 1664 | // at the target's last admission: the source's probe names the return |
| 1665 | // generation at its cold admission (§7.8, §19.1 table). |
| 1666 | func TestReturnToSourceDirtyLeavesGeneration(t *testing.T) { |
| 1667 | c := released(t) |
| 1668 | if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(c); err != nil { |
| 1669 | t.Fatal(err) |
| 1670 | } |
| 1671 | if c.Op.Phase != PhaseClaimed || c.Op.Exec != "ie-box" || c.Holder.Stamp != 3 || c.Holder.Generation != moveA { |
| 1672 | t.Fatalf("op %+v holder %+v", c.Op, c.Holder) |
| 1673 | } |
| 1674 | // src_gen may be the origin (""): the clean return then names it. |
| 1675 | d := released(t) |
| 1676 | d.Op.Journal[JournalSrcGen] = "" |
| 1677 | if err := ReturnToSource(moveA, "ie-box", true, ReturnFence{})(d); err != nil || d.Holder.Generation != "" { |
| 1678 | t.Fatalf("origin generation: %v %q", err, d.Holder.Generation) |
| 1679 | } |
| 1680 | } |
| 1681 | |
| 1682 | // declaredLost is rolledBack (restoring/releasing, exec the target) after |
| 1683 | // claim Plan 8's declare-lost transition for the target. |
| 1684 | func declaredLost(t *testing.T) *Claim { |
| 1685 | t.Helper() |
| 1686 | c := rolledBack(t) |
| 1687 | in := LostDeclaration{LossID: "loss_1", ExpectedClaimGen: c.Gen, ExpectedClaimETag: `"etag"`, |
| 1688 | HolderIncarnation: c.Holder.Incarnation, HolderStamp: c.Holder.Stamp, HolderSession: c.Holder.Session, |
| 1689 | OpID: moveA, OpKind: KindRestoring, OpPhase: PhaseReleasing, Box: "uk-box", Enrollment: "enroll-uk", |
| 1690 | QuarantineRevision: 1, FencedAt: time.Date(2026, 10, 6, 12, 0, 0, 0, time.UTC), |
| 1691 | Consequences: LossConsequences{RevokeExecution: true, RemoveParticipation: true, RecoveryIntent: []byte(`{"kind":"lost-target"}`)}} |
| 1692 | if err := declareLostTransition(in, testLossOwner{})(c); err != nil { |
| 1693 | t.Fatal(err) |
| 1694 | } |
| 1695 | if c.Op.Exec != "none" || !reflect.DeepEqual(c.Op.Participants, []string{"ie-box"}) || c.Op.Journal["loss.loss_1"] == "" || c.Op.Journal[JournalEvidenceTarget] != "" || c.Op.Phase != PhaseReleasing { |
| 1696 | t.Fatalf("fixture: %+v", c.Op) |
| 1697 | } |
| 1698 | return c |
| 1699 | } |
| 1700 | |
| 1701 | // §7.10: an operator's declaration stands in for the target's evidence. The |
| 1702 | // claim is as Plan 8's transition leaves it (exec "none", the target out of |
| 1703 | // participants, loss.<id> journalled, no evidence_target), stepped to |
| 1704 | // released by the resolve door (J16) before the source claims. |
| 1705 | func TestReturnToSourceAfterADeclaredLoss(t *testing.T) { |
| 1706 | c := declaredLost(t) |
| 1707 | before := mustJSON(t, c) |
| 1708 | // J16: the resolve door steps the claim to released first. |
| 1709 | if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) { |
| 1710 | t.Fatalf("at releasing: %v", err) |
| 1711 | } |
| 1712 | c.Op.Phase = PhaseReleased |
| 1713 | if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(c); err != nil { |
| 1714 | t.Fatal(err) |
| 1715 | } |
| 1716 | if c.Op.Exec != "ie-box" || c.Op.Phase != PhaseClaimed || c.Holder.Stamp != 3 { |
| 1717 | t.Fatalf("op %+v stamp %d", c.Op, c.Holder.Stamp) |
| 1718 | } |
| 1719 | validAfterWrite(t, c) |
| 1720 | // The same without the loss entry is refused. |
| 1721 | d := declaredLost(t) |
| 1722 | d.Op.Phase = PhaseReleased |
| 1723 | delete(d.Op.Journal, "loss.loss_1") |
| 1724 | before = mustJSON(t, d) |
| 1725 | if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(d); !errors.Is(err, ErrWrongState) || string(mustJSON(t, d)) != string(before) { |
| 1726 | t.Fatalf("without the loss entry: %v", err) |
| 1727 | } |
| 1728 | // A loss entry the claim never recorded, or one that left the target a |
| 1729 | // participant, is not the target's declaration. |
| 1730 | e := declaredLost(t) |
| 1731 | e.Op.Phase = PhaseReleased |
| 1732 | e.LossRefs = nil |
| 1733 | if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(e); !errors.Is(err, ErrWrongState) { |
| 1734 | t.Fatalf("an unrecorded loss: %v", err) |
| 1735 | } |
| 1736 | f := declaredLost(t) |
| 1737 | f.Op.Phase = PhaseReleased |
| 1738 | f.Op.Participants = []string{"ie-box", "uk-box"} |
| 1739 | if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(f); !errors.Is(err, ErrWrongState) { |
| 1740 | t.Fatalf("the target still a participant: %v", err) |
| 1741 | } |
| 1742 | } |
| 1743 | |
| 1744 | // J5: a termination fence for the target, verified by the caller before |
| 1745 | // Update (VerifyReturnFence), authorises the return when the closure finds |
| 1746 | // the same fence.<target> entry. |
| 1747 | func TestReturnToSourceWithAVerifiedFence(t *testing.T) { |
| 1748 | const ref = `{"enrollment":"0123456789abcdef0123456789abcdef","key":"control/box-fences/0123456789abcdef0123456789abcdef.json","sha256":"00"}` |
| 1749 | fenced := func() *Claim { |
| 1750 | c := rolledBack(t) |
| 1751 | c.Op.Phase, c.Op.Exec = PhaseReleased, "none" |
| 1752 | c.Op.Journal["fence.uk-box"] = ref |
| 1753 | return c |
| 1754 | } |
| 1755 | good := ReturnFence{op: moveA, box: "uk-box", ref: ref} |
| 1756 | c := fenced() |
| 1757 | if err := ReturnToSource(moveA, "ie-box", true, good)(c); err != nil { |
| 1758 | t.Fatal(err) |
| 1759 | } |
| 1760 | if c.Op.Exec != "ie-box" || c.Op.Phase != PhaseClaimed || c.Holder.Generation != "gen-7" { |
| 1761 | t.Fatalf("op %+v holder %+v", c.Op, c.Holder) |
| 1762 | } |
| 1763 | for name, f := range map[string]ReturnFence{ |
| 1764 | "unverified": {}, |
| 1765 | "another op": {op: moveB, box: "uk-box", ref: ref}, |
| 1766 | "the source's fence": {op: moveA, box: "ie-box", ref: ref}, |
| 1767 | "another journal value": {op: moveA, box: "uk-box", ref: ref + " "}, |
| 1768 | } { |
| 1769 | d := fenced() |
| 1770 | before := mustJSON(t, d) |
| 1771 | if err := ReturnToSource(moveA, "ie-box", true, f)(d); !errors.Is(err, ErrWrongState) || string(mustJSON(t, d)) != string(before) { |
| 1772 | t.Fatalf("%s: %v", name, err) |
| 1773 | } |
| 1774 | } |
| 1775 | // VerifyReturnFence verifies before it vouches: a journal entry that does |
| 1776 | // not decode, or no store, is ErrFence. |
| 1777 | op := fenced().Op |
| 1778 | if _, err := VerifyReturnFence(context.Background(), nil, nil, op, "uk-box"); !errors.Is(err, ErrFence) { |
| 1779 | t.Fatalf("no store: %v", err) |
| 1780 | } |
| 1781 | op.Journal["fence.uk-box"] = "not json" |
| 1782 | if _, err := VerifyReturnFence(context.Background(), release.NewDirStore(t.TempDir()), nil, op, "uk-box"); !errors.Is(err, ErrFence) { |
| 1783 | t.Fatalf("bad journal: %v", err) |
| 1784 | } |
| 1785 | } |
| 1786 | |
| 1787 | // K8 and J6: the release to serving from each terminal move state, once no |
| 1788 | // chore under the op remains. The executor named is the one executing. |
| 1789 | func TestReleaseMoveWaitsForObligations(t *testing.T) { |
| 1790 | states := map[string]struct { |
| 1791 | kind, phase, exec string |
| 1792 | }{ |
| 1793 | "committed landing": {KindLanding, PhaseCommitted, "uk-box"}, |
| 1794 | "opened rollback": {KindRestoring, PhaseOpened, "ie-box"}, |
| 1795 | "opened revert": {KindReverting, PhaseOpened, "ie-box"}, |
| 1796 | "the door, before the seal": {KindMoving, PhaseMoving, "ie-box"}, |
| 1797 | } |
| 1798 | for name, st := range states { |
| 1799 | c := targetWroteLanding(t) |
| 1800 | c.Op.Kind, c.Op.Phase, c.Op.Exec = st.kind, st.phase, st.exec |
| 1801 | c.Obligations = []Obligation{{ID: "o1", Op: moveA, Kind: "park-replica", Region: "ie", Retirement: "r", Owner: "region:ie"}} |
| 1802 | before := mustJSON(t, c) |
| 1803 | if err := ReleaseMove(moveA, st.exec)(c); !errors.Is(err, ErrPendingChore) || string(mustJSON(t, c)) != string(before) { |
| 1804 | t.Fatalf("%s with a chore: %v", name, err) |
| 1805 | } |
| 1806 | c.Obligations[0].Op = "another-op" |
| 1807 | if err := ReleaseMove(moveA, st.exec)(c); err != nil { |
| 1808 | t.Fatalf("%s: %v", name, err) |
| 1809 | } |
| 1810 | if c.Op.Kind != KindServing || c.Op.Exec != st.exec || c.Op.ID != "" || c.Op.Phase != "" || len(c.Op.Participants) != 0 || c.Op.Journal["released_move"] != moveA || len(c.Op.Journal) != 1 { |
| 1811 | t.Fatalf("%s: op after release %+v", name, c.Op) |
| 1812 | } |
| 1813 | if err := RequireReleasedMove(moveA)(c); err != nil { |
| 1814 | t.Fatalf("%s: %v", name, err) |
| 1815 | } |
| 1816 | validAfterWrite(t, c) |
| 1817 | } |
| 1818 | // The new states name their executor exactly. |
| 1819 | for name, st := range states { |
| 1820 | if st.kind == KindMoving { |
| 1821 | continue |
| 1822 | } |
| 1823 | c := targetWroteLanding(t) |
| 1824 | c.Op.Kind, c.Op.Phase, c.Op.Exec = st.kind, st.phase, st.exec |
| 1825 | other := "ie-box" |
| 1826 | if st.exec == "ie-box" { |
| 1827 | other = "uk-box" |
| 1828 | } |
| 1829 | if err := ReleaseMove(moveA, other)(c); !errors.Is(err, ErrWrongState) { |
| 1830 | t.Fatalf("%s released to %s: %v", name, other, err) |
| 1831 | } |
| 1832 | } |
| 1833 | // Not before those states: a revert's phase is read from op.phase (J1), |
| 1834 | // not rev_phase. |
| 1835 | for name, st := range map[string]struct{ kind, phase string }{ |
| 1836 | "landing probed": {KindLanding, "probed"}, |
| 1837 | "restoring probed": {KindRestoring, PhaseProbed}, |
| 1838 | "restoring claimed": {KindRestoring, PhaseClaimed}, |
| 1839 | "reverting probed": {KindReverting, PhaseProbed}, |
| 1840 | "reverting, rev only": {KindReverting, PhaseSealed}, |
| 1841 | "moving sealed": {KindMoving, PhaseSealed}, |
| 1842 | "moving aborting": {KindMoving, PhaseAborting}, |
| 1843 | } { |
| 1844 | c := targetWroteLanding(t) |
| 1845 | c.Op.Kind, c.Op.Phase, c.Op.Exec = st.kind, st.phase, "ie-box" |
| 1846 | if st.phase == PhaseSealed && st.kind == KindReverting { |
| 1847 | setJournal(c, JournalRevPhase, PhaseOpened) |
| 1848 | } |
| 1849 | if err := ReleaseMove(moveA, "ie-box")(c); !errors.Is(err, ErrWrongState) { |
| 1850 | t.Fatalf("%s: %v", name, err) |
| 1851 | } |
| 1852 | } |
| 1853 | } |
| 1854 | |
| 1855 | // revertingClaim is a revert as Plan 2's Revert leaves it: from the cut's |
| 1856 | // stopping (phase "", no files baseline) or from sealed (a baseline). |
| 1857 | func revertingClaim(t *testing.T, sealed bool) *Claim { |
| 1858 | t.Helper() |
| 1859 | var c *Claim |
| 1860 | if sealed { |
| 1861 | c = movingToSealed(t) |
| 1862 | } else { |
| 1863 | c = movingTestClaim(t) |
| 1864 | for _, tr := range []func(*Claim) error{ |
| 1865 | BeginMove(moveA, "ops", "ie-box", "uk-box", "uk", time.Now()), |
| 1866 | ExecToNone(moveA, "ie-box", stoppedSource, JournalEvidenceSource), |
| 1867 | } { |
| 1868 | if err := tr(c); err != nil { |
| 1869 | t.Fatal(err) |
| 1870 | } |
| 1871 | } |
| 1872 | } |
| 1873 | lane := ParkScope{Location: "moves/" + c.Host + "/" + moveA, Incarnation: c.Holder.Incarnation, EvidenceAt: time.Date(2026, 10, 6, 10, 0, 0, 0, time.UTC), RequestLimit: time.Minute} |
| 1874 | if err := Revert(moveA, "ie-box", "uk", &lane, counter())(c); err != nil { |
| 1875 | t.Fatal(err) |
| 1876 | } |
| 1877 | return c |
| 1878 | } |
| 1879 | |
| 1880 | // §7.4 and J1, J8: rev_phase runs evidence, verify, probing, probed, opened; |
| 1881 | // verify only with the files baseline the record keeps from stopped, and |
| 1882 | // straight to probing without it. op.phase mirrors probing, probed and |
| 1883 | // opened in the same write. |
| 1884 | func TestRevPhaseOrder(t *testing.T) { |
| 1885 | for _, tc := range []struct { |
| 1886 | name string |
| 1887 | sealed bool |
| 1888 | baseline bool |
| 1889 | steps [][2]string |
| 1890 | pre string |
| 1891 | }{ |
| 1892 | {"from stopped, with a baseline", true, true, [][2]string{{"evidence", "verify"}, {"verify", "probing"}, {"probing", "probed"}, {"probed", "opened"}}, PhaseSealed}, |
| 1893 | {"from stopping, no baseline", false, false, [][2]string{{"evidence", "probing"}, {"probing", "probed"}, {"probed", "opened"}}, PhaseMoving}, |
| 1894 | } { |
| 1895 | t.Run(tc.name, func(t *testing.T) { |
| 1896 | c := revertingClaim(t, tc.sealed) |
| 1897 | if c.Op.Journal[JournalRevPhase] != "evidence" || c.Op.Phase != tc.pre { |
| 1898 | t.Fatalf("fixture: %+v", c.Op) |
| 1899 | } |
| 1900 | // The other entry is refused. |
| 1901 | wrong := "probing" |
| 1902 | if !tc.baseline { |
| 1903 | wrong = "verify" |
| 1904 | } |
| 1905 | before := mustJSON(t, c) |
| 1906 | if err := SetRevPhase(moveA, "evidence", wrong, tc.baseline)(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) { |
| 1907 | t.Fatalf("evidence to %s: %v", wrong, err) |
| 1908 | } |
| 1909 | for _, step := range tc.steps { |
| 1910 | if err := SetRevPhase(moveA, step[0], step[1], tc.baseline)(c); err != nil { |
| 1911 | t.Fatalf("%s to %s: %v", step[0], step[1], err) |
| 1912 | } |
| 1913 | if c.Op.Journal[JournalRevPhase] != step[1] { |
| 1914 | t.Fatalf("rev_phase %q after %v", c.Op.Journal[JournalRevPhase], step) |
| 1915 | } |
| 1916 | want := tc.pre |
| 1917 | if step[1] != "verify" { |
| 1918 | want = step[1] |
| 1919 | } |
| 1920 | if c.Op.Phase != want || c.Op.Kind != KindReverting || c.Op.Exec != "ie-box" { |
| 1921 | t.Fatalf("op after %v: %+v", step, c.Op) |
| 1922 | } |
| 1923 | validAfterWrite(t, c) |
| 1924 | // A replay finds the phase moved. |
| 1925 | before := mustJSON(t, c) |
| 1926 | if err := SetRevPhase(moveA, step[0], step[1], tc.baseline)(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) { |
| 1927 | t.Fatalf("replay of %v: %v", step, err) |
| 1928 | } |
| 1929 | } |
| 1930 | // opened is the last phase; ReleaseMove takes it from there. |
| 1931 | if err := SetRevPhase(moveA, "opened", "probing", tc.baseline)(c); !errors.Is(err, ErrWrongState) { |
| 1932 | t.Fatalf("after opened: %v", err) |
| 1933 | } |
| 1934 | }) |
| 1935 | } |
| 1936 | // Skips, other kinds, other ops and a phase that disagrees with rev_phase. |
| 1937 | c := revertingClaim(t, true) |
| 1938 | for name, tc := range map[string]struct { |
| 1939 | op, from, to string |
| 1940 | want error |
| 1941 | }{ |
| 1942 | "skip verify to probed": {moveA, "evidence", "probed", ErrWrongState}, |
| 1943 | "skip to opened": {moveA, "evidence", "opened", ErrWrongState}, |
| 1944 | "from the wrong phase": {moveA, "verify", "probing", ErrWrongState}, |
| 1945 | "backwards": {moveA, "evidence", "evidence", ErrWrongState}, |
| 1946 | "an unknown phase": {moveA, "evidence", "stuck", ErrWrongState}, |
| 1947 | "another op": {moveB, "evidence", "verify", ErrStaleOp}, |
| 1948 | } { |
| 1949 | before := mustJSON(t, c) |
| 1950 | if err := SetRevPhase(tc.op, tc.from, tc.to, true)(c); !errors.Is(err, tc.want) || string(mustJSON(t, c)) != string(before) { |
| 1951 | t.Fatalf("%s: got %v, want %v", name, err, tc.want) |
| 1952 | } |
| 1953 | } |
| 1954 | d := revertingClaim(t, true) |
| 1955 | d.Op.Phase = PhaseProbed // disagrees with rev_phase evidence |
| 1956 | if err := SetRevPhase(moveA, "evidence", "verify", true)(d); !errors.Is(err, ErrWrongState) { |
| 1957 | t.Fatalf("op.phase ahead of rev_phase: %v", err) |
| 1958 | } |
| 1959 | e := rolledBack(t) |
| 1960 | setJournal(e, JournalRevPhase, "evidence") |
| 1961 | if err := SetRevPhase(moveA, "evidence", "verify", true)(e); !errors.Is(err, ErrWrongState) { |
| 1962 | t.Fatalf("a restoring op: %v", err) |
| 1963 | } |
| 1964 | } |
| 1965 | |
| 1966 | // J1: the source's recovery phases after claimed are mirrored onto the claim |
| 1967 | // op in order: probing (before the first recovery attempt), probed, opened. |
| 1968 | func TestRestoringPhaseOrder(t *testing.T) { |
| 1969 | c := released(t) |
| 1970 | if err := SetRestoringPhase(moveA, PhaseReleased, PhaseClaimed)(c); !errors.Is(err, ErrWrongState) { |
| 1971 | t.Fatalf("released to claimed is ReturnToSource's: %v", err) |
| 1972 | } |
| 1973 | if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(c); err != nil { |
| 1974 | t.Fatal(err) |
| 1975 | } |
| 1976 | for _, step := range [][2]string{{PhaseClaimed, PhaseProbing}, {PhaseProbing, PhaseProbed}, {PhaseProbed, PhaseOpened}} { |
| 1977 | before := mustJSON(t, c) |
| 1978 | if err := SetRestoringPhase(moveA, step[0], "nonsense")(c); !errors.Is(err, ErrWrongState) || string(mustJSON(t, c)) != string(before) { |
| 1979 | t.Fatalf("from %s to nonsense: %v", step[0], err) |
| 1980 | } |
| 1981 | if err := SetRestoringPhase(moveA, step[0], step[1])(c); err != nil { |
| 1982 | t.Fatalf("%v: %v", step, err) |
| 1983 | } |
| 1984 | if c.Op.Phase != step[1] || c.Op.Exec != "ie-box" || c.Op.Kind != KindRestoring { |
| 1985 | t.Fatalf("op after %v: %+v", step, c.Op) |
| 1986 | } |
| 1987 | if err := SetRestoringPhase(moveA, step[0], step[1])(c); !errors.Is(err, ErrWrongState) { |
| 1988 | t.Fatalf("replay %v: %v", step, err) |
| 1989 | } |
| 1990 | } |
| 1991 | d := released(t) |
| 1992 | if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(d); err != nil { |
| 1993 | t.Fatal(err) |
| 1994 | } |
| 1995 | if err := SetRestoringPhase(moveA, PhaseClaimed, PhaseProbed)(d); !errors.Is(err, ErrWrongState) { |
| 1996 | t.Fatalf("skip: %v", err) |
| 1997 | } |
| 1998 | if err := SetRestoringPhase(moveB, PhaseClaimed, PhaseProbing)(d); !errors.Is(err, ErrStaleOp) { |
| 1999 | t.Fatalf("another op: %v", err) |
| 2000 | } |
| 2001 | // The source executes from claimed on; a claim whose exec is not the |
| 2002 | // source does not step. |
| 2003 | d.Op.Exec = "uk-box" |
| 2004 | if err := SetRestoringPhase(moveA, PhaseClaimed, PhaseProbing)(d); !errors.Is(err, ErrWrongState) { |
| 2005 | t.Fatalf("exec not the source: %v", err) |
| 2006 | } |
| 2007 | } |
| 2008 | |
| 2009 | // §7.8 probed (dirty): the return lane moves/<host>/<rb id> is parked in the |
| 2010 | // source's region, queued once, and the release to serving waits for it. |
| 2011 | func TestQueueReturnLaneParkAtProbed(t *testing.T) { |
| 2012 | c := released(t) |
| 2013 | c.Obligations = nil // the target's chores are done |
| 2014 | if err := ReturnToSource(moveA, "ie-box", false, ReturnFence{})(c); err != nil { |
| 2015 | t.Fatal(err) |
| 2016 | } |
| 2017 | lane := releaseScope(c) |
| 2018 | lane.Location = "moves/" + c.Host + "/" + rbA |
| 2019 | ids := 0 |
| 2020 | newID := func() string { ids++; return fmt.Sprintf("l%d", ids) } |
| 2021 | for _, phase := range []string{PhaseClaimed, PhaseProbing} { |
| 2022 | c.Op.Phase = phase |
| 2023 | if err := QueueReturnLanePark(moveA, "ie", lane, newID)(c); !errors.Is(err, ErrWrongState) || ids != 0 { |
| 2024 | t.Fatalf("at %s: %v", phase, err) |
| 2025 | } |
| 2026 | } |
| 2027 | c.Op.Phase = PhaseProbed |
| 2028 | for name, tc := range map[string]struct { |
| 2029 | region string |
| 2030 | location string |
| 2031 | want error |
| 2032 | }{ |
| 2033 | "the target's region": {"uk", lane.Location, ErrWrongState}, |
| 2034 | "the move's lane": {"ie", "moves/" + c.Host + "/" + moveA, ErrMalformed}, |
| 2035 | "the live location": {"ie", c.Holder.Location, ErrMalformed}, |
| 2036 | "the staging prefix": {"ie", "moves/" + c.Host, ErrMalformed}, |
| 2037 | } { |
| 2038 | s := lane |
| 2039 | s.Location = tc.location |
| 2040 | before := mustJSON(t, c) |
| 2041 | if err := QueueReturnLanePark(moveA, tc.region, s, newID)(c); !errors.Is(err, tc.want) || ids != 0 || string(mustJSON(t, c)) != string(before) { |
| 2042 | t.Fatalf("%s: %v", name, err) |
| 2043 | } |
| 2044 | } |
| 2045 | if err := QueueReturnLanePark(moveA, "ie", lane, newID)(c); err != nil { |
| 2046 | t.Fatal(err) |
| 2047 | } |
| 2048 | if k := obligationKinds(c); k["park-replica/ie"] != 1 || len(c.Obligations) != 1 || c.Obligations[0].Park.Location != lane.Location || c.Obligations[0].Owner != "region:ie" { |
| 2049 | t.Fatalf("obligations: %+v", c.Obligations) |
| 2050 | } |
| 2051 | validAfterWrite(t, c) |
| 2052 | used := ids |
| 2053 | if err := QueueReturnLanePark(moveA, "ie", lane, newID)(c); err != nil || ids != used || len(c.Obligations) != 1 { |
| 2054 | t.Fatalf("replay: %v", err) |
| 2055 | } |
| 2056 | other := lane |
| 2057 | other.RequestLimit = time.Hour |
| 2058 | if err := QueueReturnLanePark(moveA, "ie", other, newID)(c); !errors.Is(err, ErrWrongState) { |
| 2059 | t.Fatalf("another scope: %v", err) |
| 2060 | } |
| 2061 | // The release waits for the lane park. |
| 2062 | if err := SetRestoringPhase(moveA, PhaseProbed, PhaseOpened)(c); err != nil { |
| 2063 | t.Fatal(err) |
| 2064 | } |
| 2065 | if err := ReleaseMove(moveA, "ie-box")(c); !errors.Is(err, ErrPendingChore) { |
| 2066 | t.Fatalf("release before the lane park: %v", err) |
| 2067 | } |
| 2068 | // Once the park engine completes the chore, a replay at probed does not |
| 2069 | // queue it again. |
| 2070 | c.Obligations = nil |
| 2071 | c.Op.Phase = PhaseProbed |
| 2072 | if err := QueueReturnLanePark(moveA, "ie", lane, newID)(c); err != nil || len(c.Obligations) != 0 || ids != used { |
| 2073 | t.Fatalf("replay after completion: %v %+v", err, c.Obligations) |
| 2074 | } |
| 2075 | c.Op.Phase = PhaseOpened |
| 2076 | if err := ReleaseMove(moveA, "ie-box")(c); err != nil { |
| 2077 | t.Fatal(err) |
| 2078 | } |
| 2079 | } |
| 2080 | |