| 1 | package hostclaim |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "errors" |
| 6 | "strconv" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | "time" |
| 10 | ) |
| 11 | |
| 12 | // The console's release of a declared-lost move target (moves spec §7.10, |
| 13 | // J16): after the declaration's claim CAS took the target out of execution |
| 14 | // and participation, one claim write rolls a landing back, steps the |
| 15 | // rollback to released, queues the target region's parks and covers its |
| 16 | // predecessors, as TargetReleased does for a target that released itself. |
| 17 | |
| 18 | const ( |
| 19 | lostOp = "mv_abcdefghijklmnopqrstuvwxyz" |
| 20 | lostRB = "rb_abcdefghijklmnopqrstuvwxyz" |
| 21 | lostID = "loss_1" |
| 22 | lostHost = "shop.example" |
| 23 | ) |
| 24 | |
| 25 | func lostTargetClaim(t *testing.T, kind, phase string) *Claim { |
| 26 | t.Helper() |
| 27 | at := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) |
| 28 | inc, err := Incarnation("bab", "jam", lostHost, at) |
| 29 | if err != nil { |
| 30 | t.Fatal(err) |
| 31 | } |
| 32 | c := &Claim{V: 2, Host: lostHost, Gen: 30, |
| 33 | Holder: &Holder{Family: "instance", Key: "apps/bab/jam/instances/" + lostHost + ".json", Account: "bab", App: "jam", CreatedAt: at, Incarnation: inc, |
| 34 | Location: "instances/" + lostHost, Generation: "mv", Stamp: 21, Session: 20, WriterBox: "uk-box", |
| 35 | Admissions: []AdmissionRecord{ |
| 36 | {Session: 15, Box: "ie-box", Generation: "src", Predecessor: "older", Region: "ie", PredecessorRegion: "ie", Mode: "cold"}, |
| 37 | {Session: 20, Box: "uk-box", Generation: "mv", Predecessor: "src", Region: "uk", PredecessorRegion: "ie", Mode: "cold", Coverage: "sealed"}, |
| 38 | {Session: 25, Box: "uk-box", Generation: "mv2", Predecessor: "mv", Region: "uk", PredecessorRegion: "uk", Mode: "cold"}, |
| 39 | }}, |
| 40 | Op: Op{Kind: kind, ID: lostOp, Phase: phase, Exec: "none", Participants: []string{"ie-box"}, |
| 41 | Intent: map[string]string{"to": "uk", "source": "ie-box", "target": "uk-box"}, |
| 42 | Journal: map[string]string{JournalSourceStamp: "15", JournalSrcGen: "src", JournalCutSession: "15", "loss." + lostID: `{"kind":"move-target-lost"}`}}, |
| 43 | Lost: []string{"uk-box"}, LostRevisions: map[string]uint64{"uk-box": 2}, LossRefs: []string{lostID}, |
| 44 | } |
| 45 | if kind == KindRestoring { |
| 46 | c.Op.Intent["rollback"] = lostRB |
| 47 | c.Op.Journal[JournalTargetStamp] = "21" |
| 48 | } |
| 49 | if err := c.Validate(); err != nil { |
| 50 | t.Fatal(err) |
| 51 | } |
| 52 | return c |
| 53 | } |
| 54 | |
| 55 | func lostRelease(c *Claim, returnLane bool) LostTargetRelease { |
| 56 | at := time.Date(2026, 10, 1, 1, 0, 0, 0, time.UTC) |
| 57 | n := 0 |
| 58 | r := LostTargetRelease{OpID: lostOp, RollbackID: lostRB, LossID: lostID, Target: "uk-box", TargetRegion: "uk", SourceRegion: "ie", |
| 59 | Location: ParkScope{Location: "instances/" + lostHost, Incarnation: c.Holder.Incarnation, EvidenceAt: at, RequestLimit: 2 * time.Minute}, |
| 60 | Lane: ParkScope{Location: "moves/" + lostHost + "/" + lostOp, Incarnation: c.Holder.Incarnation, EvidenceAt: at, RequestLimit: 2 * time.Minute}, |
| 61 | NewID: func() string { n++; return "id-" + strconv.Itoa(n) }} |
| 62 | if returnLane { |
| 63 | r.ReturnLane = &ParkScope{Location: "moves/" + lostHost + "/" + lostRB, Incarnation: c.Holder.Incarnation, EvidenceAt: at, RequestLimit: 2 * time.Minute} |
| 64 | } |
| 65 | return r |
| 66 | } |
| 67 | |
| 68 | func claimJSON(t *testing.T, c *Claim) string { |
| 69 | t.Helper() |
| 70 | b, err := json.Marshal(c) |
| 71 | if err != nil { |
| 72 | t.Fatal(err) |
| 73 | } |
| 74 | return string(b) |
| 75 | } |
| 76 | |
| 77 | func parksOf(c *Claim) map[string]Obligation { |
| 78 | out := map[string]Obligation{} |
| 79 | for _, o := range c.Obligations { |
| 80 | key := o.Kind + "|" + o.Region + "|" + o.Box |
| 81 | if o.Park != nil { |
| 82 | key += "|" + o.Park.Location |
| 83 | } |
| 84 | out[key] = o |
| 85 | } |
| 86 | return out |
| 87 | } |
| 88 | |
| 89 | func TestReleaseLostTargetFromLanding(t *testing.T) { |
| 90 | c := lostTargetClaim(t, KindLanding, "probing") |
| 91 | if err := ReleaseLostTarget(lostRelease(c, false))(c); err != nil { |
| 92 | t.Fatal(err) |
| 93 | } |
| 94 | if c.Op.Kind != KindRestoring || c.Op.Phase != PhaseReleased || c.Op.Intent["rollback"] != lostRB || c.Op.Exec != "none" { |
| 95 | t.Fatalf("op = %+v", c.Op) |
| 96 | } |
| 97 | if c.Op.Journal[JournalTargetStamp] != "21" || c.Op.Journal[journalLostTargetReleased] != lostID { |
| 98 | t.Fatalf("journal = %v", c.Op.Journal) |
| 99 | } |
| 100 | parks := parksOf(c) |
| 101 | loc, ok := parks["park-replica|uk||instances/"+lostHost] |
| 102 | if !ok || loc.Owner != "region:uk" { |
| 103 | t.Fatalf("location park = %+v (%v)", loc, parks) |
| 104 | } |
| 105 | if _, ok := parks["park-replica|uk||moves/"+lostHost+"/"+lostOp]; !ok { |
| 106 | t.Fatalf("target lane park missing: %v", parks) |
| 107 | } |
| 108 | if len(parks) != 2 { |
| 109 | t.Fatalf("parks = %v", parks) |
| 110 | } |
| 111 | for _, o := range c.Obligations { |
| 112 | if o.Kind == "park-local" || o.Box != "" { |
| 113 | t.Fatalf("a target with no installation in inventory acquired local cleanup: %+v", o) |
| 114 | } |
| 115 | } |
| 116 | // The target's own predecessor is covered by the park; the source's |
| 117 | // return admission later is born covered the same way. |
| 118 | for _, a := range c.Holder.Admissions { |
| 119 | if a.PredecessorRegion == "uk" && !a.Covered { |
| 120 | t.Fatalf("uk predecessor not covered: %+v", a) |
| 121 | } |
| 122 | if a.PredecessorRegion == "ie" && a.Covered && a.Session != 15 { |
| 123 | t.Fatalf("ie predecessor covered: %+v", a) |
| 124 | } |
| 125 | } |
| 126 | if err := c.Validate(); err != nil { |
| 127 | t.Fatal(err) |
| 128 | } |
| 129 | // The source's claimed CAS takes it from here: the declaration is the |
| 130 | // target's authority (targetGaveUp). |
| 131 | if err := ReturnToSource(lostOp, "ie-box", true, ReturnFence{})(c); err != nil { |
| 132 | t.Fatalf("return to source after the release: %v", err) |
| 133 | } |
| 134 | } |
| 135 | |
| 136 | func TestReleaseLostTargetIsIdempotent(t *testing.T) { |
| 137 | c := lostTargetClaim(t, KindLanding, "probing") |
| 138 | r := lostRelease(c, false) |
| 139 | if err := ReleaseLostTarget(r)(c); err != nil { |
| 140 | t.Fatal(err) |
| 141 | } |
| 142 | once := claimJSON(t, c) |
| 143 | if err := ReleaseLostTarget(lostRelease(c, false))(c); err != nil { |
| 144 | t.Fatal(err) |
| 145 | } |
| 146 | if claimJSON(t, c) != once { |
| 147 | t.Fatal("a replay changed the claim") |
| 148 | } |
| 149 | // A replay after the source has claimed, or after a chore was acked, |
| 150 | // is still a no-op. |
| 151 | if err := ReturnToSource(lostOp, "ie-box", true, ReturnFence{})(c); err != nil { |
| 152 | t.Fatal(err) |
| 153 | } |
| 154 | c.Obligations = c.Obligations[:1] |
| 155 | claimed := claimJSON(t, c) |
| 156 | if err := ReleaseLostTarget(lostRelease(c, false))(c); err != nil { |
| 157 | t.Fatal(err) |
| 158 | } |
| 159 | if claimJSON(t, c) != claimed { |
| 160 | t.Fatal("a replay after the source claimed changed the claim") |
| 161 | } |
| 162 | } |
| 163 | |
| 164 | func TestReleaseLostTargetFromReleasing(t *testing.T) { |
| 165 | c := lostTargetClaim(t, KindRestoring, PhaseReleasing) |
| 166 | if err := ReleaseLostTarget(lostRelease(c, false))(c); err != nil { |
| 167 | t.Fatal(err) |
| 168 | } |
| 169 | if c.Op.Phase != PhaseReleased || len(parksOf(c)) != 2 { |
| 170 | t.Fatalf("op %+v parks %v", c.Op, parksOf(c)) |
| 171 | } |
| 172 | } |
| 173 | |
| 174 | // A target that released dirty and then died before its return: its own |
| 175 | // release queued the location park already; the declaration adds the |
| 176 | // partial return lane's park in the source region and the move's lane. |
| 177 | func TestReleaseLostTargetAfterDirtyRelease(t *testing.T) { |
| 178 | c := lostTargetClaim(t, KindRestoring, PhaseReleased) |
| 179 | c.Op.Exec = "" |
| 180 | c.Op.Journal[JournalEvidenceTarget] = "moves/x/stopped-target.json" |
| 181 | at := time.Date(2026, 10, 1, 0, 30, 0, 0, time.UTC) |
| 182 | c.Obligations = []Obligation{{ID: "loc", Op: lostOp, Kind: "park-replica", Region: "uk", Retirement: "r-loc", Owner: "region:uk", |
| 183 | Park: &ParkScope{Location: "instances/" + lostHost, Incarnation: c.Holder.Incarnation, EvidenceAt: at, RequestLimit: time.Minute}}} |
| 184 | if err := ReleaseLostTarget(lostRelease(c, true))(c); err != nil { |
| 185 | t.Fatal(err) |
| 186 | } |
| 187 | parks := parksOf(c) |
| 188 | if len(parks) != 3 || parks["park-replica|uk||instances/"+lostHost].ID != "loc" { |
| 189 | t.Fatalf("parks = %v", parks) |
| 190 | } |
| 191 | lane, ok := parks["park-replica|ie||moves/"+lostHost+"/"+lostRB] |
| 192 | if !ok || lane.Owner != "region:ie" { |
| 193 | t.Fatalf("return lane park = %+v (%v)", lane, parks) |
| 194 | } |
| 195 | if c.Op.Journal[journalReturnLane] != "moves/"+lostHost+"/"+lostRB { |
| 196 | t.Fatalf("journal = %v", c.Op.Journal) |
| 197 | } |
| 198 | // The return lane's own probed-time queue is then a no-op. |
| 199 | c.Op.Phase = PhaseProbed |
| 200 | before := len(c.Obligations) |
| 201 | if err := QueueReturnLanePark(lostOp, "ie", *lostRelease(c, true).ReturnLane, func() string { return "x" })(c); err != nil { |
| 202 | t.Fatal(err) |
| 203 | } |
| 204 | if len(c.Obligations) != before { |
| 205 | t.Fatal("return lane queued twice") |
| 206 | } |
| 207 | } |
| 208 | |
| 209 | func TestReleaseLostTargetRefusals(t *testing.T) { |
| 210 | for name, tc := range map[string]struct { |
| 211 | mutate func(*Claim, *LostTargetRelease) |
| 212 | refusal error |
| 213 | }{ |
| 214 | "no loss ref": {func(c *Claim, _ *LostTargetRelease) { c.LossRefs = nil }, ErrWrongState}, |
| 215 | "no loss journal": {func(c *Claim, _ *LostTargetRelease) { delete(c.Op.Journal, "loss."+lostID) }, ErrWrongState}, |
| 216 | "target participates": {func(c *Claim, _ *LostTargetRelease) { c.Op.Participants = []string{"ie-box", "uk-box"} }, ErrWrongState}, |
| 217 | "target executes": {func(c *Claim, _ *LostTargetRelease) { c.Op.Exec = "uk-box" }, ErrWrongState}, |
| 218 | "target not lost": {func(c *Claim, _ *LostTargetRelease) { c.Lost, c.LostRevisions = nil, nil }, ErrWrongState}, |
| 219 | "other target": {func(_ *Claim, r *LostTargetRelease) { r.Target = "ie-box" }, ErrWrongState}, |
| 220 | "other region": {func(_ *Claim, r *LostTargetRelease) { r.TargetRegion = "de" }, ErrWrongState}, |
| 221 | "source region is to": {func(_ *Claim, r *LostTargetRelease) { r.SourceRegion = "uk" }, ErrMalformed}, |
| 222 | "other op": {func(_ *Claim, r *LostTargetRelease) { r.OpID = "mv_bbbbbbbbbbbbbbbbbbbbbbbbbb" }, ErrStaleOp}, |
| 223 | "committed": {func(c *Claim, _ *LostTargetRelease) { c.Op.Phase = PhaseCommitted }, ErrWrongState}, |
| 224 | "bad rollback": {func(_ *Claim, r *LostTargetRelease) { r.RollbackID = "rb_x" }, ErrMalformed}, |
| 225 | "wrong lane": {func(_ *Claim, r *LostTargetRelease) { r.Lane.Location = "moves/" + lostHost }, ErrMalformed}, |
| 226 | "wrong return lane": {func(c *Claim, r *LostTargetRelease) { s := r.Lane; r.ReturnLane = &s }, ErrMalformed}, |
| 227 | "wrong location": {func(_ *Claim, r *LostTargetRelease) { r.Location.Location = "instances/" + lostHost + "/x" }, ErrMalformed}, |
| 228 | "no ids": {func(_ *Claim, r *LostTargetRelease) { r.NewID = nil }, ErrMalformed}, |
| 229 | "moving (before flip)": {func(c *Claim, _ *LostTargetRelease) { c.Op.Kind, c.Op.Phase = KindMoving, PhaseSealed }, ErrWrongState}, |
| 230 | } { |
| 231 | t.Run(name, func(t *testing.T) { |
| 232 | c := lostTargetClaim(t, KindLanding, "probing") |
| 233 | r := lostRelease(c, false) |
| 234 | tc.mutate(c, &r) |
| 235 | before := claimJSON(t, c) |
| 236 | err := ReleaseLostTarget(r)(c) |
| 237 | if !errors.Is(err, tc.refusal) { |
| 238 | t.Fatalf("err = %v, want %v", err, tc.refusal) |
| 239 | } |
| 240 | if claimJSON(t, c) != before { |
| 241 | t.Fatal("a refused release changed the claim") |
| 242 | } |
| 243 | }) |
| 244 | } |
| 245 | // Another rollback already on the claim. |
| 246 | c := lostTargetClaim(t, KindRestoring, PhaseReleasing) |
| 247 | r := lostRelease(c, false) |
| 248 | r.RollbackID = "rb_bbbbbbbbbbbbbbbbbbbbbbbbbb" |
| 249 | if err := ReleaseLostTarget(r)(c); !errors.Is(err, ErrWrongState) { |
| 250 | t.Fatalf("other rollback = %v", err) |
| 251 | } |
| 252 | if !strings.HasPrefix(lostRB, "rb_") { |
| 253 | t.Fatal("fixture") |
| 254 | } |
| 255 | } |
| 256 | |
| 257 | // D1: the lanes are checked against the holder's incarnation, as every |
| 258 | // other writer of a move lane does, and a trailing slash is normalized. |
| 259 | func TestReleaseLostTargetLaneIncarnation(t *testing.T) { |
| 260 | other, err := Incarnation("bab", "jam", lostHost, time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)) |
| 261 | if err != nil { |
| 262 | t.Fatal(err) |
| 263 | } |
| 264 | for name, mutate := range map[string]func(*Claim, *LostTargetRelease){ |
| 265 | "lane of another incarnation": func(_ *Claim, r *LostTargetRelease) { r.Lane.Incarnation = other }, |
| 266 | "return lane of another incarnation": func(c *Claim, r *LostTargetRelease) { |
| 267 | s := lostRelease(c, true).ReturnLane |
| 268 | s.Incarnation = other |
| 269 | r.ReturnLane = s |
| 270 | }, |
| 271 | } { |
| 272 | t.Run(name, func(t *testing.T) { |
| 273 | c := lostTargetClaim(t, KindLanding, "probing") |
| 274 | r := lostRelease(c, false) |
| 275 | mutate(c, &r) |
| 276 | before := claimJSON(t, c) |
| 277 | if err := ReleaseLostTarget(r)(c); !errors.Is(err, ErrMalformed) { |
| 278 | t.Fatalf("err = %v, want %v", err, ErrMalformed) |
| 279 | } |
| 280 | if claimJSON(t, c) != before { |
| 281 | t.Fatal("a refused release changed the claim") |
| 282 | } |
| 283 | }) |
| 284 | } |
| 285 | c := lostTargetClaim(t, KindLanding, "probing") |
| 286 | r := lostRelease(c, true) |
| 287 | r.Lane.Location += "/" |
| 288 | r.ReturnLane.Location += "/" |
| 289 | if err := ReleaseLostTarget(r)(c); err != nil { |
| 290 | t.Fatal(err) |
| 291 | } |
| 292 | parks := parksOf(c) |
| 293 | if _, ok := parks["park-replica|uk||moves/"+lostHost+"/"+lostOp]; !ok { |
| 294 | t.Fatalf("normalized lane park missing: %v", parks) |
| 295 | } |
| 296 | if _, ok := parks["park-replica|ie||moves/"+lostHost+"/"+lostRB]; !ok || c.Op.Journal[journalReturnLane] != "moves/"+lostHost+"/"+lostRB { |
| 297 | t.Fatalf("normalized return lane park missing: %v %v", parks, c.Op.Journal) |
| 298 | } |
| 299 | } |
| 300 | |
| 301 | // D2: a target that released dirty and then died owes its return lane's |
| 302 | // park; a release without it is refused and leaves the claim as found. |
| 303 | func TestReleaseLostTargetReleasedNeedsReturnLane(t *testing.T) { |
| 304 | c := lostTargetClaim(t, KindRestoring, PhaseReleased) |
| 305 | c.Op.Exec = "" |
| 306 | c.Op.Journal[JournalEvidenceTarget] = "moves/x/stopped-target.json" |
| 307 | before := claimJSON(t, c) |
| 308 | if err := ReleaseLostTarget(lostRelease(c, false))(c); !errors.Is(err, ErrMalformed) { |
| 309 | t.Fatalf("err = %v, want %v", err, ErrMalformed) |
| 310 | } |
| 311 | if claimJSON(t, c) != before { |
| 312 | t.Fatal("a refused release changed the claim") |
| 313 | } |
| 314 | } |
| 315 | |
| 316 | // D3: Readmit removes the box from lost without looking at the op, so a |
| 317 | // retried release whose CAS landed is still a no-op afterwards. |
| 318 | func TestReleaseLostTargetReplayAfterReadmit(t *testing.T) { |
| 319 | c := lostTargetClaim(t, KindLanding, "probing") |
| 320 | if err := ReleaseLostTarget(lostRelease(c, false))(c); err != nil { |
| 321 | t.Fatal(err) |
| 322 | } |
| 323 | if err := Readmit("uk-box", 2)(c); err != nil { |
| 324 | t.Fatal(err) |
| 325 | } |
| 326 | once := claimJSON(t, c) |
| 327 | if err := ReleaseLostTarget(lostRelease(c, false))(c); err != nil { |
| 328 | t.Fatalf("replay after readmit: %v", err) |
| 329 | } |
| 330 | if claimJSON(t, c) != once { |
| 331 | t.Fatal("a replay changed the claim") |
| 332 | } |
| 333 | } |
| 334 | |
| 335 | func TestReleaseLostTargetRetainsInventoryBoundLocal(t *testing.T) { |
| 336 | for _, kind := range []string{KindLanding, KindRestoring} { |
| 337 | t.Run(kind, func(t *testing.T) { |
| 338 | phase := PhaseReleasing |
| 339 | if kind == KindLanding { |
| 340 | phase = PhaseProbing |
| 341 | } |
| 342 | c := lostTargetClaim(t, kind, phase) |
| 343 | c.Inventory = map[string]InventoryEntry{"uk-box": {Paths: map[string]uint64{"root": 21}}} |
| 344 | if err := ReleaseLostTarget(lostRelease(c, false))(c); err != nil { |
| 345 | t.Fatal(err) |
| 346 | } |
| 347 | var local Obligation |
| 348 | for _, o := range c.Obligations { |
| 349 | if o.Kind == "park-local" && o.Box == "uk-box" { |
| 350 | local = o |
| 351 | } |
| 352 | } |
| 353 | if local.ID == "" || local.Op != lostOp || local.Retirement == "" { |
| 354 | t.Fatal("target inventory's retirement obligation missing") |
| 355 | } |
| 356 | before := claimJSON(t, c) |
| 357 | if err := ReleaseLostTarget(lostRelease(c, false))(c); err != nil { |
| 358 | t.Fatal(err) |
| 359 | } |
| 360 | if claimJSON(t, c) != before { |
| 361 | t.Fatal("retry lost or replaced local chore") |
| 362 | } |
| 363 | if err := ReturnToSource(lostOp, "ie-box", true, ReturnFence{})(c); err != nil { |
| 364 | t.Fatal("pending target cleanup blocked source recovery", err) |
| 365 | } |
| 366 | c.Op.Phase, c.Op.Exec = PhaseOpened, "ie-box" |
| 367 | c.Obligations = []Obligation{local} |
| 368 | if err := ReleaseMove(lostOp, "ie-box")(c); err == nil { |
| 369 | t.Fatal("move released with target-local retirement pending") |
| 370 | } |
| 371 | if err := CompleteObligation(local.Op, local.ID)(c); err != nil { |
| 372 | t.Fatal(err) |
| 373 | } |
| 374 | if err := ReleaseMove(lostOp, "ie-box")(c); err != nil { |
| 375 | t.Fatal("retired local still blocks release", err) |
| 376 | } |
| 377 | }) |
| 378 | } |
| 379 | } |
| 380 | |
amadan