| 1 | package hostlife |
| 2 | |
| 3 | import ( |
| 4 | "crypto/sha256" |
| 5 | "fmt" |
| 6 | "testing" |
| 7 | ) |
| 8 | |
| 9 | // Astra's round 1 on the two-box plan (rev 4): a direct test for every |
| 10 | // guard the round found unproven, the phase matrix stopped between single |
| 11 | // actor transitions, and claim §11's additional witnesses that rev 3 left |
| 12 | // outside the model. Each test names the rule whose mutation makes it |
| 13 | // fail. |
| 14 | |
| 15 | // stepUntil advances the world one actor at a time and stops the moment |
| 16 | // done holds: the tick's own work (completions, stalls, the two-box |
| 17 | // step), then each live box's worker, obligation runner, reconcile and |
| 18 | // passes, the executor's wake, then the recovery pass, with done checked |
| 19 | // after every single one. A fault injected after it returns lands |
| 20 | // exactly between two transitions. |
| 21 | func (w *world) stepUntil(t *testing.T, max int, what string, done func() bool) bool { |
| 22 | t.Helper() |
| 23 | if done() { |
| 24 | return true |
| 25 | } |
| 26 | for i := 0; i < max; i++ { |
| 27 | w.now++ |
| 28 | w.completeDue(false) |
| 29 | if p := w.recPend; p != nil && p.at <= w.now { |
| 30 | w.recPend = nil |
| 31 | if w.rec == nil { |
| 32 | w.rec = p.rec |
| 33 | } |
| 34 | } |
| 35 | w.runStalled() |
| 36 | if w.perRegion > 1 { |
| 37 | w.twoBoxStep() |
| 38 | } |
| 39 | if done() { |
| 40 | return true |
| 41 | } |
| 42 | for _, b := range w.liveBoxes() { |
| 43 | for _, act := range []func(){ |
| 44 | func() { w.worker(b) }, func() { w.obligationRunner(b) }, func() { w.reconcile(b) }, |
| 45 | func() { |
| 46 | if len(b.plans) == 0 { |
| 47 | w.passPlan(b, "config") |
| 48 | w.passPlan(b, "adopt") |
| 49 | } |
| 50 | w.passExec(b) |
| 51 | }, |
| 52 | w.wakeExec, |
| 53 | } { |
| 54 | act() |
| 55 | if len(w.violations) > 0 { |
| 56 | return false |
| 57 | } |
| 58 | if done() { |
| 59 | return true |
| 60 | } |
| 61 | } |
| 62 | } |
| 63 | w.recoveryPass() |
| 64 | w.checkStep() |
| 65 | if len(w.violations) > 0 { |
| 66 | return false |
| 67 | } |
| 68 | if done() { |
| 69 | return true |
| 70 | } |
| 71 | } |
| 72 | w.violate("INV-3 (witness): %s not reached within %d steps: %s", what, max, w.state()) |
| 73 | return false |
| 74 | } |
| 75 | |
| 76 | // atPhase: the relocation is exactly in phase p (not beyond it), as |
| 77 | // 1b-next names it: an initial failover at pinning with PinFailover set |
| 78 | // is in starting (rev 7, Astra round 4 finding 2). |
| 79 | func (w *world) atPhase(p string) func() bool { |
| 80 | return func() bool { |
| 81 | c := w.claim |
| 82 | return c.Op.Kind == "relocating" && relocPhaseName(c.Op) == p && c.Op.Stuck == "" |
| 83 | } |
| 84 | } |
| 85 | |
| 86 | // relocPhaseName is the operation's phase as 1b-next names it: the |
| 87 | // model's pinning with op.PinFailover set is production's starting. |
| 88 | func relocPhaseName(op opState) string { |
| 89 | if op.Phase == "pinning" && op.PinFailover { |
| 90 | return "starting" |
| 91 | } |
| 92 | return op.Phase |
| 93 | } |
| 94 | |
| 95 | // liveIn counts region r's live boxes. |
| 96 | func (w *world) liveIn(r int) int { return len(w.regionBoxes(r)) } |
| 97 | |
| 98 | // ---------- finding 6: the termination matrix, exact ---------- |
| 99 | |
| 100 | // TestL18TerminationAtEveryExactPhase (L18 "termination during every |
| 101 | // relocation phase", rev 4): the relocation is stopped exactly in each |
| 102 | // phase of §5.8's table, probe-failed included, between two single |
| 103 | // actor transitions; the phase and the region's live topology are |
| 104 | // asserted immediately before the participant is terminated; no spare |
| 105 | // exists beforehand, and replacements come only from the modelled group |
| 106 | // after the loss. One-box region: the failover's target (the box the |
| 107 | // group launched) is terminated. Two-box region: the drain's source or |
| 108 | // target is. |
| 109 | func TestL18TerminationAtEveryExactPhase(t *testing.T) { |
| 110 | phases := []string{"starting", "releasing", "pinning", "installing", "installed", "probing", "probe-failed", "probed"} |
| 111 | for _, boxes := range []int{1, 2} { |
| 112 | for _, who := range []string{"source", "target"} { |
| 113 | for _, phase := range phases { |
| 114 | if boxes == 1 && (who == "source" || phase == "releasing") { |
| 115 | continue // a one-box region relocates only by failover, whose source is fenced at its start; its starting is PinFailover's wait (rev 7) |
| 116 | } |
| 117 | if phase == "probe-failed" && who == "source" { |
| 118 | continue // the source has released before any probe; its loss is the pinning case |
| 119 | } |
| 120 | name := fmt.Sprintf("%s/%s@%s", map[int]string{1: "one-box region", 2: "two-box region"}[boxes], who, phase) |
| 121 | t.Run(name, func(t *testing.T) { |
| 122 | w := regionWorld(boxes) |
| 123 | from := w.serveAndPublish(t) |
| 124 | w.appWrite(from) |
| 125 | w.timerOn(from) |
| 126 | var target func() *box |
| 127 | if boxes == 2 { |
| 128 | w.drainTo(t, nil) |
| 129 | tgt := w.boxByID(w.claim.Op.Tgt) |
| 130 | tgt.FailApp = phase == "probe-failed" |
| 131 | target = func() *box { return w.boxByID(w.claim.Op.Tgt) } |
| 132 | } else { |
| 133 | w.terminate(from) |
| 134 | if !w.stepUntil(t, 400, "the failover started with its target", func() bool { c := w.claim; return c.Op.Kind == "relocating" && c.Op.Tgt >= 0 }) { |
| 135 | w.mustHold(t) |
| 136 | } |
| 137 | if c := w.claim; relocPhaseName(c.Op) != "starting" { |
| 138 | t.Fatalf("the failover did not start in starting (PinFailover unset): %s", w.state()) |
| 139 | } |
| 140 | w.boxByID(w.claim.Op.Tgt).FailApp = phase == "probe-failed" |
| 141 | target = func() *box { return w.boxByID(w.claim.Op.Tgt) } |
| 142 | } |
| 143 | if !w.stepUntil(t, 2000, "relocation exactly at "+phase, w.atPhase(phase)) { |
| 144 | w.mustHold(t) |
| 145 | } |
| 146 | // immediately before the injection: the exact phase, and the |
| 147 | // region's live topology (no spare) |
| 148 | if c := w.claim; relocPhaseName(c.Op) != phase { |
| 149 | t.Fatalf("not at %s: %s", phase, w.state()) |
| 150 | } |
| 151 | if c := w.claim; phase == "pinning" && (c.Op.PinFailover || c.Op.Pin != nil && c.Op.Exec >= 0) { |
| 152 | t.Fatalf("pinning is after PinFailover and before installing: %s", w.state()) |
| 153 | } |
| 154 | if n := w.liveIn(0); n != boxes { |
| 155 | t.Fatalf("region 0 has %d live boxes before the injection, want %d", n, boxes) |
| 156 | } |
| 157 | victim := from |
| 158 | if who == "target" { |
| 159 | victim = target() |
| 160 | } |
| 161 | if victim == nil || victim.Destroyed { |
| 162 | t.Fatalf("no live %s at %s: %s", who, phase, w.state()) |
| 163 | } |
| 164 | w.terminate(victim) |
| 165 | w.servingOn(t, nil) |
| 166 | w.finish(t) |
| 167 | w.mustHold(t) |
| 168 | if c := w.claim; c.Op.Exec == victim.ID { |
| 169 | t.Fatalf("serving on the terminated %s", who) |
| 170 | } |
| 171 | }) |
| 172 | } |
| 173 | } |
| 174 | } |
| 175 | } |
| 176 | |
| 177 | // ---------- finding 2: stuck refuses admission ---------- |
| 178 | |
| 179 | // TestStuckSourceRefusesAdmission (reloc-stuck-refuses-admission): three |
| 180 | // targets die in starting, so the exhausted fence leaves the live source |
| 181 | // op.exec with its ready row and the operation stuck. A request then |
| 182 | // reaches the source, whose app has stopped: no writer is admitted. |
| 183 | func TestStuckSourceRefusesAdmission(t *testing.T) { |
| 184 | w := twoBoxWitness() |
| 185 | from := w.boxes[0] |
| 186 | w.serve(from, 1, app0) |
| 187 | w.running(t, from) |
| 188 | b30, b31 := w.extraBox(0, 30), w.extraBox(0, 31) |
| 189 | w.userRelocate() |
| 190 | for _, x := range []*box{w.boxByID(w.claim.Op.Tgt), b30, b31, w.boxes[3]} { |
| 191 | if x == nil || x.Destroyed || x.ID != w.claim.Op.Tgt { |
| 192 | continue |
| 193 | } |
| 194 | w.terminate(x) |
| 195 | w.reach(t, 60, "the dead target fenced", func() bool { return contains(w.claim.Lost, x.ID) }, w.act(from)) |
| 196 | } |
| 197 | if c := w.claim; c.Op.Stuck == "" || c.Op.Exec != from.ID || c.Op.Phase != "starting" || from.Row == nil || from.Row.Status != "ready" { |
| 198 | t.Fatalf("not stuck in starting with the live source op.exec and its row ready: %s", w.state()) |
| 199 | } |
| 200 | from.Procs = nil // the app stopped; the next request wakes it |
| 201 | before := w.stats["admissions: cold"] + w.stats["admissions: warm"] |
| 202 | w.drive(30, never, func() { w.wake(from) }) |
| 203 | w.mustHold(t) |
| 204 | if n := w.stats["admissions: cold"] + w.stats["admissions: warm"] - before; n != 0 || hasMain(from) { |
| 205 | t.Fatalf("a stuck operation admitted %d writers on its source (running %v)", n, hasMain(from)) |
| 206 | } |
| 207 | } |
| 208 | |
| 209 | // ---------- finding 3: the sole box is checked in an HA region ---------- |
| 210 | |
| 211 | // TestHASoleBoxIsCheckedLikeAnyOther (ha-sole-box-eligibility): in a |
| 212 | // one-box region with region HA on, the only box is refused when it is |
| 213 | // unenrolled, lacks the secrets key, runs an old executable, is on lost or |
| 214 | // owes a park-local for the host; in the one-box shape (region HA off) |
| 215 | // the region's box is chosen as today. |
| 216 | func TestHASoleBoxIsCheckedLikeAnyOther(t *testing.T) { |
| 217 | for _, why := range []string{"unenrolled", "no key", "old executable", "lost", "owes a park-local"} { |
| 218 | t.Run(why, func(t *testing.T) { |
| 219 | w := regionWorld(1) |
| 220 | b := w.boxes[0] |
| 221 | w.serve(w.boxes[1], 1, app0) // the claim lives in region 1; region 0 is the create's |
| 222 | switch why { |
| 223 | case "unenrolled": |
| 224 | b.Enrolled = false |
| 225 | case "no key": |
| 226 | b.HasKey = false |
| 227 | case "old executable": |
| 228 | b.Old = true |
| 229 | case "lost": |
| 230 | w.claim.Lost = addUniq(w.claim.Lost, b.ID) |
| 231 | case "owes a park-local": |
| 232 | w.claim.Obl = append(w.claim.Obl, obligation{ID: "pl", Op: "old", Kind: "park-local", Box: b.ID, Region: -1, Ret: "old"}) |
| 233 | } |
| 234 | if got := w.chooseBox(0, app0); got != nil { |
| 235 | t.Fatalf("region HA chose its sole box %d although it is %s", got.ID, why) |
| 236 | } |
| 237 | }) |
| 238 | } |
| 239 | t.Run("one-box shape keeps today's pick", func(t *testing.T) { |
| 240 | w := newWitness() |
| 241 | w.boxes[0].Enrolled = false |
| 242 | if got := w.chooseBox(0, app0); got != w.boxes[0] { |
| 243 | t.Fatalf("the one-box shape did not pick the region's box: %v", got) |
| 244 | } |
| 245 | }) |
| 246 | } |
| 247 | |
| 248 | // ---------- finding 4: the acknowledgement barrier at every transition ---------- |
| 249 | |
| 250 | // TestReplacementWithoutAcknowledgementHoldsEachTransition |
| 251 | // (reloc-exec-ack-barrier): the target dies at installing and the spare |
| 252 | // that replaces it has not acknowledged. Driving the installation's steps |
| 253 | // without the spare's acknowledgement, the operation stays at installing |
| 254 | // (ToInstalled's requireAcks); once the spare acknowledges, it proceeds. |
| 255 | func TestReplacementWithoutAcknowledgementHoldsEachTransition(t *testing.T) { |
| 256 | w := twoBoxWitness() |
| 257 | from, to := w.boxes[0], w.boxes[3] |
| 258 | w.serve(from, 1, app0) |
| 259 | w.running(t, from) |
| 260 | w.published(t, from) |
| 261 | w.drainTo(t, to) |
| 262 | spare := w.extraBox(0, 30) |
| 263 | if !w.reach(t, 1500, "installing", w.atPhase("installing"), w.act(from, to), w.recoveryPass) { |
| 264 | w.mustHold(t) |
| 265 | } |
| 266 | w.terminate(to) |
| 267 | w.reach(t, 60, "replaced at installing", func() bool { return w.claim.Op.Exec == spare.ID }, w.act(from)) |
| 268 | if contains(w.claim.Op.Acks, spare.ID) { |
| 269 | t.Fatalf("the spare acknowledged before it ran") |
| 270 | } |
| 271 | // the spare's installation steps, without its acknowledgement |
| 272 | spare.ExecUntil = w.localNow(spare) + execTTL |
| 273 | for i := 0; i < 40; i++ { |
| 274 | w.tick() |
| 275 | w.relocInstalling(spare, w.claim.clone()) |
| 276 | } |
| 277 | w.mustHold(t) |
| 278 | if c := w.claim; c.Op.Phase != "installing" { |
| 279 | t.Fatalf("left installing without the spare's acknowledgement: %s", w.state()) |
| 280 | } |
| 281 | w.servingOn(t, spare) |
| 282 | w.finish(t) |
| 283 | w.mustHold(t) |
| 284 | } |
| 285 | |
| 286 | // ---------- finding 5: deadlines and delays from the fence record ---------- |
| 287 | |
| 288 | // TestPinWaitsForEveryFenceObservation (reloc-pin-delay-every-fence): the |
| 289 | // source's release is journaled, then the target is terminated and |
| 290 | // replaced by a spare before any pin. The pin waits out the target's |
| 291 | // fence observation plus the delay, a release notwithstanding. |
| 292 | func TestPinWaitsForEveryFenceObservation(t *testing.T) { |
| 293 | w := twoBoxWitness() |
| 294 | from, to := w.boxes[0], w.boxes[3] |
| 295 | w.serve(from, 1, app0) |
| 296 | w.running(t, from) |
| 297 | w.published(t, from) |
| 298 | w.drainTo(t, to) |
| 299 | spare := w.extraBox(0, 30) |
| 300 | if !w.stepUntil(t, 1500, "the source's release journaled", func() bool { c := w.claim; return c.Op.Release != nil && c.Op.Pin == nil }) { |
| 301 | w.mustHold(t) |
| 302 | } |
| 303 | w.terminate(to) |
| 304 | ok := w.stepUntil(t, 400, "the pin", func() bool { |
| 305 | c := w.claim |
| 306 | if c.Op.Pin != nil && len(c.Op.Observed) > 0 && w.now < latestObservation(c.Op)+failoverPinDelay { |
| 307 | t.Fatalf("pinned at %d, before the target's observation %d plus %d", w.now, latestObservation(c.Op), failoverPinDelay) |
| 308 | } |
| 309 | return c.Op.Pin != nil |
| 310 | }) |
| 311 | if !ok { |
| 312 | w.mustHold(t) |
| 313 | } |
| 314 | if !contains(w.claim.Lost, to.ID) || w.claim.Op.Tgt != spare.ID { |
| 315 | t.Fatalf("the target was not replaced before the pin: %s", w.state()) |
| 316 | } |
| 317 | w.servingOn(t, spare) |
| 318 | w.finish(t) |
| 319 | w.mustHold(t) |
| 320 | } |
| 321 | |
| 322 | // TestDeadTargetDeadlineFromObservation (reloc-deadline-from-observed-at): |
| 323 | // the drain's target dies at installing while no worker of the region can |
| 324 | // observe it (the source is frozen) for longer than the deadline, and the |
| 325 | // group launches no spare. The first observation writes the fence record; |
| 326 | // the operation waits a whole deadline from it before it is stuck. |
| 327 | func TestDeadTargetDeadlineFromObservation(t *testing.T) { |
| 328 | w := twoBoxWitness() |
| 329 | from, to := w.boxes[0], w.boxes[3] |
| 330 | w.serve(from, 1, app0) |
| 331 | w.running(t, from) |
| 332 | w.published(t, from) |
| 333 | w.drainTo(t, to) |
| 334 | w.oneBoxRegion[0] = true // no replacement launches: no spare |
| 335 | if !w.reach(t, 1500, "installing", w.atPhase("installing"), w.act(from, to), w.recoveryPass) { |
| 336 | w.mustHold(t) |
| 337 | } |
| 338 | w.terminate(to) |
| 339 | from.Frozen = true |
| 340 | w.drive(releaseDeadline+20, never, w.act(w.boxes[1], w.boxes[2])) |
| 341 | from.Frozen = false |
| 342 | w.stepUntil(t, 40, "the fence record", func() bool { return to.FenceObservedAt > 0 }) |
| 343 | seen := w.now |
| 344 | w.drive(releaseDeadline-5, never, w.act(from), w.recoveryPass) |
| 345 | if w.claim.Op.Stuck != "" { |
| 346 | t.Fatalf("stuck %d steps after the observation at %d, before its deadline (terminated at %d)", w.now-seen, seen, to.TerminatedAt) |
| 347 | } |
| 348 | w.reach(t, 60, "stuck after the deadline", func() bool { return w.claim.Op.Stuck != "" }, w.act(from), w.recoveryPass) |
| 349 | w.mustHold(t) |
| 350 | } |
| 351 | |
| 352 | // ---------- finding 7: the five guards, each discriminated ---------- |
| 353 | |
| 354 | // TestOnlyTheExecutorWritesP (reloc-exec-only-installs): at installing, |
| 355 | // only the source (not op.exec) runs: nothing is written under P and no |
| 356 | // local install is made on it. |
| 357 | func TestOnlyTheExecutorWritesP(t *testing.T) { |
| 358 | w := twoBoxWitness() |
| 359 | from, to := w.boxes[0], w.boxes[3] |
| 360 | w.serve(from, 1, app0) |
| 361 | w.running(t, from) |
| 362 | w.published(t, from) |
| 363 | w.drainTo(t, to) |
| 364 | if !w.stepUntil(t, 1500, "installing", w.atPhase("installing")) { |
| 365 | w.mustHold(t) |
| 366 | } |
| 367 | p := w.claim.Op.Prepared |
| 368 | for _, k := range w.buckets[0].keys(dataKey(w.claim.Holder.Location, p, "")) { |
| 369 | w.buckets[0].objs[k] = nil // whatever the target began is set aside: only the source acts now |
| 370 | delete(w.buckets[0].objs, k) |
| 371 | } |
| 372 | w.drive(40, never, w.act(from)) |
| 373 | if ks := w.buckets[0].keys(dataKey(w.claim.Holder.Location, p, "")); len(ks) > 0 || from.receipt != nil || from.Files != nil && from.Files.Gen == p { |
| 374 | t.Fatalf("the source, not op.exec, wrote P or installed it: %v", ks) |
| 375 | } |
| 376 | } |
| 377 | |
| 378 | // TestInstallAppliesThePinnedManifest (files-pin-exact): a newer manifest |
| 379 | // of the dead executor lands between the pin and the install; the install |
| 380 | // applies the pinned version, never the newest. |
| 381 | func TestInstallAppliesThePinnedManifest(t *testing.T) { |
| 382 | w := twoBoxWitness() |
| 383 | from, to := w.boxes[0], w.boxes[3] |
| 384 | w.serve(from, 1, app0) |
| 385 | w.running(t, from) |
| 386 | w.published(t, from) |
| 387 | loc := w.claim.Holder.Location |
| 388 | late := w.timerOn(from) |
| 389 | w.holdKey(loc + "files/") |
| 390 | w.publishFiles(from) // the newer manifest, carrying the timer, held |
| 391 | w.hold = nil |
| 392 | w.terminate(from) |
| 393 | if !w.stepUntil(t, 400, "the pin journaled", func() bool { c := w.claim; return c.Op.Pin != nil }) { |
| 394 | w.mustHold(t) |
| 395 | } |
| 396 | if n := w.completeHeld(loc + "files/"); n == 0 { |
| 397 | t.Fatalf("no held manifest to land") |
| 398 | } |
| 399 | if !w.servingOn(t, to) { |
| 400 | w.mustHold(t) |
| 401 | } |
| 402 | if contains(to.Timers, late) { |
| 403 | t.Fatalf("the install applied the manifest that landed after the pin") |
| 404 | } |
| 405 | } |
| 406 | |
| 407 | // TestFailoverPinWaitsOutTheDelay (reloc-failover-pin-delay): no pin |
| 408 | // before the fence's observed_at plus the delay. |
| 409 | func TestFailoverPinWaitsOutTheDelay(t *testing.T) { |
| 410 | w := twoBoxWitness() |
| 411 | from, to := w.boxes[0], w.boxes[3] |
| 412 | w.serve(from, 1, app0) |
| 413 | w.running(t, from) |
| 414 | w.published(t, from) |
| 415 | w.terminate(from) |
| 416 | ok := w.stepUntil(t, 400, "the pin", func() bool { |
| 417 | c := w.claim |
| 418 | if c.Op.Pin != nil && w.now < c.Op.FenceAt+failoverPinDelay { |
| 419 | t.Fatalf("pinned at %d, before the fence's observation %d plus %d", w.now, c.Op.FenceAt, failoverPinDelay) |
| 420 | } |
| 421 | return c.Op.Kind == "relocating" && c.Op.Pin != nil |
| 422 | }) |
| 423 | if !ok { |
| 424 | w.mustHold(t) |
| 425 | } |
| 426 | w.servingOn(t, to) |
| 427 | w.finish(t) |
| 428 | w.mustHold(t) |
| 429 | } |
| 430 | |
| 431 | // TestFailoverPinWaitsForTheAcknowledgement (reloc-failover-ack-before-pin; |
| 432 | // rev 5, Astra round 2 finding 1; 1b-next rev 19 PinFailover): the |
| 433 | // executor's newer files manifest is still in flight when it is |
| 434 | // terminated. The target is named, but its controller does not run, so |
| 435 | // it has not acknowledged; the pin waits past the delay, the manifest |
| 436 | // lands, and only then does the target acknowledge. The pin names the |
| 437 | // newer manifest. The mutation pins before the acknowledgement, and the |
| 438 | // pin misses it. |
| 439 | func TestFailoverPinWaitsForTheAcknowledgement(t *testing.T) { |
| 440 | w := twoBoxWitness() |
| 441 | from := w.boxes[0] |
| 442 | w.serve(from, 1, app0) |
| 443 | w.running(t, from) |
| 444 | w.published(t, from) |
| 445 | spare := w.extraBox(0, 30) // a second live box of the region: the worker while the target's controller is down |
| 446 | c := w.claim |
| 447 | loc, gen := c.Holder.Location, c.Holder.Gen |
| 448 | from.Files.Env++ // a newer files state: the admin token rotated |
| 449 | newer := manifestKey(loc, gen, from.pubSeq+1) |
| 450 | w.holdKey(newer) |
| 451 | w.publishFiles(from) |
| 452 | w.hold = nil |
| 453 | if from.pubSeq == 0 || manifestKey(loc, gen, from.pubSeq) != newer { |
| 454 | t.Fatalf("the newer manifest was not issued: %s", w.state()) |
| 455 | } |
| 456 | w.terminate(from) |
| 457 | if !w.stepUntil(t, 400, "the failover's target named", func() bool { |
| 458 | c := w.claim |
| 459 | return c.Op.Kind == "relocating" && (c.Op.Tgt >= 0 || c.Op.Pin != nil) |
| 460 | }) { |
| 461 | w.mustHold(t) |
| 462 | } |
| 463 | if w.claim.Op.Pin != nil { |
| 464 | t.Fatalf("pinned before a target was named and acknowledged: %s", w.state()) |
| 465 | } |
| 466 | tgt := w.boxByID(w.claim.Op.Tgt) |
| 467 | worker := spare |
| 468 | if tgt == spare { |
| 469 | worker = w.boxes[3] |
| 470 | } |
| 471 | w.drive(failoverPinDelay+20, never, w.act(worker)) // the target's controller does not run |
| 472 | if contains(w.claim.Op.Acks, tgt.ID) { |
| 473 | t.Fatalf("the target acknowledged without running") |
| 474 | } |
| 475 | if p := w.claim.Op.Pin; p != nil { |
| 476 | t.Fatalf("pinned %s before the target acknowledged: %s", p.ManKey, w.state()) |
| 477 | } |
| 478 | if n := w.completeHeld(newer); n != 1 { |
| 479 | t.Fatalf("held manifest PUTs completed: %d, want 1", n) |
| 480 | } |
| 481 | if !w.reach(t, 400, "the pin", func() bool { return w.claim.Op.Pin != nil }, w.act(worker, tgt)) { |
| 482 | w.mustHold(t) |
| 483 | } |
| 484 | if p := w.claim.Op.Pin; p.ManKey != newer { |
| 485 | t.Fatalf("the pin names %s, want the manifest that landed while the target had not acknowledged, %s", p.ManKey, newer) |
| 486 | } |
| 487 | w.servingOn(t, tgt) |
| 488 | w.finish(t) |
| 489 | w.mustHold(t) |
| 490 | } |
| 491 | |
| 492 | // convertedMove serves the host on box 0 of a two-box witness world with |
| 493 | // a spare box 30 in region 0, starts a move to region 1, and terminates |
| 494 | // the source before the flip, so the regional worker converts the move |
| 495 | // to a relocation at pinning with no target (moves §7.10). before runs |
| 496 | // before the move starts. |
| 497 | func (w *world) convertedMove(t *testing.T, before func()) *box { |
| 498 | src := w.boxes[0] |
| 499 | w.serve(src, 1, app0) |
| 500 | w.running(t, src) |
| 501 | w.published(t, src) |
| 502 | if before != nil { |
| 503 | before() |
| 504 | } |
| 505 | w.moveTo(1) |
| 506 | if w.claim.Op.Kind != "moving" { |
| 507 | t.Fatalf("no move: %s", w.state()) |
| 508 | } |
| 509 | tgt := w.boxByID(w.claim.Op.Tgt) |
| 510 | w.reach(t, 100, "staging begins", func() bool { |
| 511 | c := w.claim |
| 512 | return c.Op.Kind == "moving" && c.Op.Phase == "preparing" && contains(c.Regions, tgt.Region) |
| 513 | }, w.act(src, tgt)) |
| 514 | w.terminate(src) |
| 515 | if !w.stepUntil(t, 400, "the move converted", func() bool { |
| 516 | c := w.claim |
| 517 | return c.Op.Kind == "relocating" && c.Op.FenceBox == src.ID |
| 518 | }) { |
| 519 | w.mustHold(t) |
| 520 | } |
| 521 | if c := w.claim; c.Op.PinFailover || c.Op.Pin != nil { |
| 522 | t.Fatalf("not a converted move at pinning without a pin: %s", w.state()) |
| 523 | } |
| 524 | return src |
| 525 | } |
| 526 | |
| 527 | // TestConvertedMovePinsBeforeAcknowledgement (rev 6, Astra round 3 |
| 528 | // finding 1; rev 7, round 4 finding 1): a converted move enters pinning |
| 529 | // directly, as in 1b-next, and Worker.pinning names its target first, |
| 530 | // then selects and journals the pin, and checks the acknowledgements only |
| 531 | // for ToInstalling. The named target's controller does not run while the |
| 532 | // region's other box works: the pin is journaled without the target's |
| 533 | // acknowledgement, and installing waits for it. |
| 534 | func TestConvertedMovePinsBeforeAcknowledgement(t *testing.T) { |
| 535 | w := twoBoxWitness() |
| 536 | spare := w.extraBox(0, 30) |
| 537 | w.convertedMove(t, nil) |
| 538 | if !w.stepUntil(t, 400, "the target named", func() bool { c := w.claim; return c.Op.Tgt >= 0 || c.Op.Pin != nil }) { |
| 539 | w.mustHold(t) |
| 540 | } |
| 541 | c := w.claim |
| 542 | if c.Op.Pin != nil { |
| 543 | t.Fatalf("pinned before a target was named: %s", w.state()) |
| 544 | } |
| 545 | tgt := w.boxByID(c.Op.Tgt) |
| 546 | worker := spare |
| 547 | if tgt == spare { |
| 548 | worker = w.boxes[3] |
| 549 | } |
| 550 | w.drive(failoverPinDelay+20, never, w.act(worker)) // the target's controller does not run |
| 551 | c = w.claim |
| 552 | if contains(c.Op.Acks, tgt.ID) { |
| 553 | t.Fatalf("the target acknowledged without running") |
| 554 | } |
| 555 | if c.Op.Pin == nil { |
| 556 | t.Fatalf("the converted move waited for an acknowledgement before its pin: %s", w.state()) |
| 557 | } |
| 558 | if c.Op.Phase != "pinning" || c.Op.Exec >= 0 { |
| 559 | t.Fatalf("installing without the target's acknowledgement: %s", w.state()) |
| 560 | } |
| 561 | w.servingOn(t, tgt) |
| 562 | w.finish(t) |
| 563 | w.mustHold(t) |
| 564 | } |
| 565 | |
| 566 | // TestConvertedMoveWaitsForATargetBeforeItsPin (reloc-name-before-pin; |
| 567 | // rev 7, Astra round 4 finding 1): the source's newer files manifest is |
| 568 | // in flight when the move starts and when the source is terminated. No |
| 569 | // box of the region is eligible (the spare has no key), so the converted |
| 570 | // move names no target and selects no pin; the manifest lands; the |
| 571 | // spare becomes eligible, is named, and the pin names the newer |
| 572 | // manifest. The mutation pins first, without the manifest. |
| 573 | func TestConvertedMoveWaitsForATargetBeforeItsPin(t *testing.T) { |
| 574 | w := twoBoxWitness() |
| 575 | peer := w.boxes[3] |
| 576 | var newer string |
| 577 | w.convertedMove(t, func() { |
| 578 | src, c := w.boxes[0], w.claim |
| 579 | src.Files.Env++ // a newer files state: the admin token rotated |
| 580 | newer = manifestKey(c.Holder.Location, c.Holder.Gen, src.pubSeq+1) |
| 581 | w.holdKey(newer) |
| 582 | w.publishFiles(src) |
| 583 | w.hold = nil |
| 584 | peer.HasKey = false // the region's other box may work, never take the host |
| 585 | }) |
| 586 | if newer == "" || w.boxes[0].pubSeq == 0 { |
| 587 | t.Fatalf("the newer manifest was not issued") |
| 588 | } |
| 589 | w.drive(failoverPinDelay+20, never, w.act(peer)) |
| 590 | if c := w.claim; c.Op.Pin != nil || c.Op.Tgt >= 0 { |
| 591 | t.Fatalf("pinned or named with no eligible box: %s", w.state()) |
| 592 | } |
| 593 | if n := w.completeHeld(newer); n != 1 { |
| 594 | t.Fatalf("held manifest PUTs completed: %d, want 1", n) |
| 595 | } |
| 596 | peer.HasKey = true |
| 597 | if !w.reach(t, 400, "the pin", func() bool { return w.claim.Op.Pin != nil }, w.act(peer)) { |
| 598 | w.mustHold(t) |
| 599 | } |
| 600 | if p := w.claim.Op.Pin; p.ManKey != newer { |
| 601 | t.Fatalf("the pin names %s, want the manifest that landed while no box was eligible, %s", p.ManKey, newer) |
| 602 | } |
| 603 | w.servingOn(t, peer) |
| 604 | w.finish(t) |
| 605 | w.mustHold(t) |
| 606 | } |
| 607 | |
| 608 | // convertedMoveTargetStopped converts a move (convertedMove: the source, |
| 609 | // box 0, is terminated before the flip) and stops the move's own target, |
| 610 | // the one participant the conversion left, before it acknowledges; it |
| 611 | // returns that box. Stopped is not terminated: nothing fences it, and |
| 612 | // it never acknowledges the converted operation. |
| 613 | func (w *world) convertedMoveTargetStopped(t *testing.T) *box { |
| 614 | w.convertedMove(t, nil) |
| 615 | c := w.claim |
| 616 | if len(c.Op.Parts) != 1 || len(c.Op.Acks) != 0 || c.Op.Tgt >= 0 { |
| 617 | t.Fatalf("not a converted move whose one participant is the move's target: %s", w.state()) |
| 618 | } |
| 619 | mt := w.boxByID(c.Op.Parts[0]) |
| 620 | if mt == nil || mt.Region == c.Op.SrcRegion { |
| 621 | t.Fatalf("the converted move's participant %d is not the move's target in another region: %s", c.Op.Parts[0], w.state()) |
| 622 | } |
| 623 | w.stop(mt) |
| 624 | return mt |
| 625 | } |
| 626 | |
| 627 | // TestConvertedMoveDropsTheMoveTargetFromTheBarrier |
| 628 | // (reloc-name-drops-move-target; final review Important 1, 1b-next |
| 629 | // 151261fe): the move's source is terminated before the flip, and the |
| 630 | // move's own target in region 1, the one participant the conversion |
| 631 | // left, is stopped before it acknowledges. Naming the same-region target |
| 632 | // takes the move's target off the participants in the same CAS, and the |
| 633 | // relocation reaches installing and serves while that box stays |
| 634 | // stopped. The mutation keeps it on the barrier: the relocation waits at |
| 635 | // pinning for ever. |
| 636 | func TestConvertedMoveDropsTheMoveTargetFromTheBarrier(t *testing.T) { |
| 637 | w := twoBoxWitness() |
| 638 | mt := w.convertedMoveTargetStopped(t) |
| 639 | if !w.reach(t, 400, "the target named", func() bool { return w.claim.Op.Tgt >= 0 }, w.actAll(), w.recoveryPass) { |
| 640 | w.mustHold(t) |
| 641 | } |
| 642 | if c := w.claim; contains(c.Op.Parts, mt.ID) || len(c.Op.Parts) != 1 || c.Op.Parts[0] != c.Op.Tgt { |
| 643 | t.Fatalf("the move's target %d is still on the barrier: parts=%v target=%d", mt.ID, c.Op.Parts, c.Op.Tgt) |
| 644 | } |
| 645 | if !w.relocPhase(t, "installing", w.actAll(), w.recoveryPass) { |
| 646 | w.mustHold(t) |
| 647 | } |
| 648 | if !mt.Destroyed { |
| 649 | t.Fatalf("the move's target started again") |
| 650 | } |
| 651 | w.servingOn(t, nil) |
| 652 | w.finish(t) |
| 653 | w.mustHold(t) |
| 654 | } |
| 655 | |
| 656 | // TestFailoverStartWaitsForAdmission (reloc-failover-start-guard; rev |
| 657 | // 7, Astra round 4 finding 3): with the app's admission closed, a |
| 658 | // terminated serving executor's failover does not start (Worker.Start's |
| 659 | // Admission.Register); once it reopens, it starts and serves on the |
| 660 | // region's other box. |
| 661 | func TestFailoverStartWaitsForAdmission(t *testing.T) { |
| 662 | w := twoBoxWitness() |
| 663 | from := w.boxes[0] |
| 664 | w.serve(from, 1, app0) |
| 665 | w.running(t, from) |
| 666 | w.published(t, from) |
| 667 | ad := w.admit[app0] |
| 668 | ad.Closed = true |
| 669 | w.terminate(from) |
| 670 | w.drive(60, never, w.actAll(), w.recoveryPass) |
| 671 | if c := w.claim; c.Op.Kind != "serving" || contains(c.Lost, from.ID) { |
| 672 | t.Fatalf("a failover started while the app's admission was closed: %s", w.state()) |
| 673 | } |
| 674 | ad.Closed = false |
| 675 | if to := w.failedOver(t, from); to == nil || to.Region != from.Region { |
| 676 | t.Fatalf("no failover in the region: %s", w.state()) // the peer, or the group's replacement, whichever choose() prefers |
| 677 | } |
| 678 | w.finish(t) |
| 679 | w.mustHold(t) |
| 680 | } |
| 681 | |
| 682 | // TestFailoverStartWaitsForParkReplica (reloc-failover-start-guard; rev |
| 683 | // 7, Astra round 4 finding 3): a park-replica chore is outstanding in the |
| 684 | // region when the serving executor is terminated. RelocationStartGuard |
| 685 | // refuses the failover's start until the chore is done; the source's own |
| 686 | // park-local (the fenced source) would not have. |
| 687 | func TestFailoverStartWaitsForParkReplica(t *testing.T) { |
| 688 | w := twoBoxWitness() |
| 689 | from := w.boxes[0] |
| 690 | w.serve(from, 1, app0) |
| 691 | w.running(t, from) |
| 692 | w.published(t, from) |
| 693 | w.claim.Obl = append(w.claim.Obl, w.scoped(obligation{ID: "pr", Op: "old-op", Kind: "park-replica", Region: 0, Box: -1, Prefix: "moves/" + host + "/", Quiet: w.now + 20}, w.now, 1)) |
| 694 | w.claim.Obl = append(w.claim.Obl, obligation{ID: "pl", Op: "old-op", Kind: "park-local", Box: from.ID, Region: -1, Ret: "old-op"}) |
| 695 | from.DeleteOnTermination = false // the fence keeps the park-local open |
| 696 | w.terminate(from) |
| 697 | if !w.stepUntil(t, 400, "the failover's start", func() bool { |
| 698 | c := w.claim |
| 699 | if c.Op.Kind == "relocating" { |
| 700 | for _, o := range c.Obl { |
| 701 | if o.Kind == "park-replica" && o.Region == 0 { |
| 702 | t.Fatalf("the failover started with a park-replica outstanding in its region: %s", w.state()) |
| 703 | } |
| 704 | } |
| 705 | return true |
| 706 | } |
| 707 | return false |
| 708 | }) { |
| 709 | w.mustHold(t) |
| 710 | } |
| 711 | if !w.owesLocal(w.claim, from) { |
| 712 | t.Fatalf("the fenced source's park-local was not kept open: %s", w.state()) |
| 713 | } |
| 714 | if to := w.failedOver(t, from); to == nil || to.Region != from.Region { |
| 715 | t.Fatalf("no failover in the region: %s", w.state()) |
| 716 | } |
| 717 | w.finish(t) |
| 718 | w.mustHold(t) |
| 719 | } |
| 720 | |
| 721 | // TestFailoverTargetLossAroundPinFailover (rev 7, Astra round 4 finding |
| 722 | // 2): an initial failover's target is terminated while PinFailover is |
| 723 | // pending (production's starting) and, in a second run, after it has |
| 724 | // cleared and before installing (pinning). In a two-box region with a |
| 725 | // spare, the target is replaced in the same phase and the host serves. |
| 726 | func TestFailoverTargetLossAroundPinFailover(t *testing.T) { |
| 727 | for _, phase := range []string{"starting", "pinning"} { |
| 728 | t.Run(phase, func(t *testing.T) { |
| 729 | w := twoBoxWitness() |
| 730 | from := w.boxes[0] |
| 731 | w.extraBox(0, 30) |
| 732 | w.serve(from, 1, app0) |
| 733 | w.running(t, from) |
| 734 | w.published(t, from) |
| 735 | w.terminate(from) |
| 736 | if !w.stepUntil(t, 400, "the failover exactly at "+phase, w.atPhase(phase)) { |
| 737 | w.mustHold(t) |
| 738 | } |
| 739 | c := w.claim |
| 740 | if (phase == "starting") != c.Op.PinFailover || c.Op.Exec >= 0 || c.Op.Tgt < 0 { |
| 741 | t.Fatalf("not at %s: %s", phase, w.state()) |
| 742 | } |
| 743 | victim := w.boxByID(c.Op.Tgt) |
| 744 | w.terminate(victim) |
| 745 | if !w.stepUntil(t, 400, "the target replaced", func() bool { |
| 746 | c := w.claim |
| 747 | return contains(c.Lost, victim.ID) && c.Op.Tgt >= 0 && c.Op.Tgt != victim.ID |
| 748 | }) { |
| 749 | w.mustHold(t) |
| 750 | } |
| 751 | if got := relocPhaseName(w.claim.Op); got != phase { |
| 752 | t.Fatalf("the replacement left %s for %s: %s", phase, got, w.state()) |
| 753 | } |
| 754 | w.servingOn(t, nil) |
| 755 | w.finish(t) |
| 756 | w.mustHold(t) |
| 757 | }) |
| 758 | } |
| 759 | } |
| 760 | |
| 761 | // TestPendingTargetFenceHoldsThePhase (reloc-fence-pending-blocks; rev |
| 762 | // 8, Astra round 5): a drain's target has acknowledged and is terminated, |
| 763 | // once in starting and once in pinning before a pin; the region's group |
| 764 | // launches no replacement, so the fence cannot be consumed until its |
| 765 | // deadline. Until then the source's worker makes no regional transition: |
| 766 | // starting does not go to releasing, and pinning selects no pin. Under |
| 767 | // the mutation the worker goes on into the phase switch and this test |
| 768 | // fails. Past the deadline the fence is consumed and the operation is |
| 769 | // stuck, as FenceTargetWithoutSpare makes it. |
| 770 | func TestPendingTargetFenceHoldsThePhase(t *testing.T) { |
| 771 | for _, phase := range []string{"starting", "pinning"} { |
| 772 | t.Run(phase, func(t *testing.T) { |
| 773 | w := twoBoxWitness() |
| 774 | from, to := w.boxes[0], w.boxes[3] |
| 775 | w.serve(from, 1, app0) |
| 776 | w.running(t, from) |
| 777 | w.published(t, from) |
| 778 | w.oneBoxRegion[0] = true // no replacement launches: no spare |
| 779 | w.drainTo(t, to) |
| 780 | if phase == "starting" { |
| 781 | // the target's controller acknowledges (AcknowledgeExecution); |
| 782 | // the source has not yet, so the operation stays in starting |
| 783 | to.ExecUntil = w.localNow(to) + execTTL |
| 784 | w.recordAck(to, w.claim.clone()) |
| 785 | } else if !w.stepUntil(t, 1500, "the target acknowledged at "+phase, func() bool { |
| 786 | c := w.claim |
| 787 | return contains(c.Op.Acks, to.ID) && w.atPhase(phase)() && c.Op.Pin == nil |
| 788 | }) { |
| 789 | w.mustHold(t) |
| 790 | } |
| 791 | if c := w.claim; !contains(c.Op.Acks, to.ID) || !w.atPhase(phase)() || c.Op.Pin != nil { |
| 792 | t.Fatalf("not at %s with the target acknowledged: %s", phase, w.state()) |
| 793 | } |
| 794 | w.oneBoxRegion[0] = false |
| 795 | w.terminate(to) |
| 796 | w.oneBoxRegion[0] = true |
| 797 | for i := 0; i < releaseDeadline-10; i++ { |
| 798 | w.tick(w.act(from)) |
| 799 | c := w.claim |
| 800 | if relocPhaseName(c.Op) != phase || c.Op.Pin != nil { |
| 801 | t.Fatalf("the phase moved on while the target's fence was pending (step %d): %s", i, w.state()) |
| 802 | } |
| 803 | if contains(c.Lost, to.ID) { |
| 804 | t.Fatalf("the dead target was fenced with no spare before its deadline: %s", w.state()) |
| 805 | } |
| 806 | } |
| 807 | w.mustHold(t) |
| 808 | if w.stats["relocation: phase held, a participant's fence is pending"] == 0 { |
| 809 | t.Fatalf("no worker step held the phase") |
| 810 | } |
| 811 | if !w.reach(t, 100, "the fence consumed into stuck", func() bool { c := w.claim; return contains(c.Lost, to.ID) && c.Op.Stuck != "" }, w.act(from)) { |
| 812 | w.mustHold(t) |
| 813 | } |
| 814 | }) |
| 815 | } |
| 816 | } |
| 817 | |
| 818 | // TestReleaseRefusedAfterACrashBeforeTheFlush (reloc-release-needs-flush; |
| 819 | // rev 9, Astra round 6): a drain's source is in releasing, its last write |
| 820 | // not yet uploaded, when its agent crashes: the session's replicator is |
| 821 | // gone and nothing recorded a flush for this stop. No release is |
| 822 | // journaled and no pin selected; at the release deadline the operation |
| 823 | // is stuck, and the operator's resolve returns the host to its source. |
| 824 | // The mutation journals the remote position with Flushed false. |
| 825 | func TestReleaseRefusedAfterACrashBeforeTheFlush(t *testing.T) { |
| 826 | w := twoBoxWitness() |
| 827 | from, to := w.boxes[0], w.boxes[3] |
| 828 | w.serve(from, 1, app0) |
| 829 | w.running(t, from) |
| 830 | w.published(t, from) |
| 831 | w.drainTo(t, to) |
| 832 | w.appWrite(from) |
| 833 | w.holdKey(w.claim.Holder.Location) // the flush's upload does not land |
| 834 | if !w.stepUntil(t, 1500, "the source waiting for its flush", func() bool { |
| 835 | return w.claim.Op.Phase == "releasing" && w.stats["release: waits for the flush"] > 0 |
| 836 | }) { |
| 837 | w.mustHold(t) |
| 838 | } |
| 839 | w.crash(from) |
| 840 | w.hold = nil |
| 841 | for i := 0; i < releaseDeadline+20; i++ { |
| 842 | w.tick(w.act(from, to)) |
| 843 | c := w.claim |
| 844 | if c.Op.Release != nil || c.Op.Pin != nil { |
| 845 | t.Fatalf("a release or a pin without the source's flush (step %d): %s", i, w.state()) |
| 846 | } |
| 847 | if c.Op.Stuck != "" { |
| 848 | break |
| 849 | } |
| 850 | } |
| 851 | if c := w.claim; c.Op.Stuck == "" || c.Op.Phase != "releasing" { |
| 852 | t.Fatalf("not stuck in releasing at the release deadline: %s", w.state()) |
| 853 | } |
| 854 | w.finish(t) |
| 855 | w.mustHold(t) |
| 856 | if c := w.claim; c.Op.Kind != "serving" || c.Op.Exec != from.ID { |
| 857 | t.Fatalf("the operator's resolve did not return the host to its source: %s", w.state()) |
| 858 | } |
| 859 | } |
| 860 | |
| 861 | // TestReleaseReplaysTheRecordedFlush (reloc-release-needs-flush; rev 9, |
| 862 | // Astra round 6; rev 11, round 8): the source's release sequence |
| 863 | // completes, records its flushed positions (ReleasePositions), and root |
| 864 | // persists the stop receipt; its agent then crashes before the release |
| 865 | // is journaled. The restarted agent has no replicator for the session, |
| 866 | // but the record and the receipt are durable: the release is replayed |
| 867 | // with the recorded positions, Flushed, and the drain completes. The |
| 868 | // mutation of reloc-release-needs-flush records nothing and journals the |
| 869 | // remote position with Flushed false. |
| 870 | func TestReleaseReplaysTheRecordedFlush(t *testing.T) { |
| 871 | w := twoBoxWitness() |
| 872 | from, to := w.boxes[0], w.boxes[3] |
| 873 | w.serve(from, 1, app0) |
| 874 | w.running(t, from) |
| 875 | w.published(t, from) |
| 876 | w.drainTo(t, to) |
| 877 | w.appWrite(from) |
| 878 | if !w.stepUntil(t, 1500, "releasing", w.atPhase("releasing")) { |
| 879 | w.mustHold(t) |
| 880 | } |
| 881 | var r *releaseEntry |
| 882 | for i := 0; i < 60 && r == nil; i++ { |
| 883 | r = w.relocRelease(from, w.claim.clone()) // steps 1 to 3, the record and the receipt, not yet journaled |
| 884 | w.tick() |
| 885 | } |
| 886 | if r == nil || !r.Flushed { |
| 887 | t.Fatalf("the release sequence did not complete: %v %s", r, w.state()) |
| 888 | } |
| 889 | w.crash(from) |
| 890 | if !w.reach(t, 60, "the release journaled", func() bool { return w.claim.Op.Release != nil }, w.act(from, to)) { |
| 891 | w.mustHold(t) |
| 892 | } |
| 893 | got := w.claim.Op.Release |
| 894 | if !got.Flushed || got.FinalTX != r.FinalTX || got.ManifestSeq != r.ManifestSeq { |
| 895 | t.Fatalf("the journaled release %+v is not the recorded one %+v", got, r) |
| 896 | } |
| 897 | w.servingOn(t, to) |
| 898 | w.finish(t) |
| 899 | w.mustHold(t) |
| 900 | } |
| 901 | |
| 902 | // probeFailedTarget drains the host from box 0 to box 3, whose app |
| 903 | // fails its probe, with a spare box 30 for the nested relocation, and |
| 904 | // stops once the operation is in probe-failed with box 3 op.exec and no |
| 905 | // release journaled. |
| 906 | func (w *world) probeFailedTarget(t *testing.T) (from, to *box) { |
| 907 | w.claimC1 = true // these tests are of the failed target's release, which claim ask C1 makes possible (finding 4) |
| 908 | from, to = w.boxes[0], w.boxes[3] |
| 909 | to.FailApp = true |
| 910 | w.serve(from, 1, app0) |
| 911 | w.running(t, from) |
| 912 | w.published(t, from) |
| 913 | w.drainTo(t, to) |
| 914 | w.extraBox(0, 30) |
| 915 | if !w.stepUntil(t, 1500, "probe-failed on the target", func() bool { |
| 916 | c := w.claim |
| 917 | return c.Op.Phase == "probe-failed" && c.Op.Exec == to.ID && c.Op.Release == nil |
| 918 | }) { |
| 919 | w.mustHold(t) |
| 920 | } |
| 921 | return from, to |
| 922 | } |
| 923 | |
| 924 | // resolveOnly is the operator's resolve of a stuck relocation and |
| 925 | // nothing else (the destroyed-local declaration is left out). |
| 926 | func (w *world) resolveOnly() { |
| 927 | if c := w.claim; c != nil && c.Op.Kind == "relocating" && c.Op.Stuck != "" { |
| 928 | w.operator() |
| 929 | } |
| 930 | } |
| 931 | |
| 932 | // TestOperatorResumesASlowRelease (operator-stop-unrecoverable-only; rev |
| 933 | // 10, Astra round 7 finding 2): the failed target's final manifest PUT is |
| 934 | // not acknowledged before the release deadline, so the operation is |
| 935 | // stuck. Its replicator and files are intact: the operator's resolve |
| 936 | // resumes the release rather than stopping the box; once the PUT lands |
| 937 | // the release is journaled and the nested relocation serves. The |
| 938 | // mutation stops the box. |
| 939 | func TestOperatorResumesASlowRelease(t *testing.T) { |
| 940 | w := twoBoxWitness() |
| 941 | _, to := w.probeFailedTarget(t) |
| 942 | h := w.claim.Holder |
| 943 | held := dataKey(h.Location, h.Gen, "files/") |
| 944 | w.holdKey(held) |
| 945 | if !w.reach(t, 2*releaseDeadline, "stuck at the release deadline", func() bool { return w.claim.Op.Stuck != "" }, w.actAll(), w.recoveryPass) { |
| 946 | w.mustHold(t) |
| 947 | } |
| 948 | if c := w.claim; c.Op.Phase != "probe-failed" || c.Op.Release != nil { |
| 949 | t.Fatalf("not stuck in probe-failed without a release: %s", w.state()) |
| 950 | } |
| 951 | w.resolveOnly() |
| 952 | if c := w.claim; to.Destroyed || c.Op.Stuck != "" || c.Op.Phase != "probe-failed" || c.Op.Exec != to.ID { |
| 953 | t.Fatalf("the resolve did not resume the target's release (stopped %v): %s", to.Destroyed, w.state()) |
| 954 | } |
| 955 | w.hold = nil |
| 956 | w.completeHeld(held) |
| 957 | w.servingOn(t, nil) |
| 958 | if w.stats["operator stopped a target that cannot release"] != 0 || w.stats["release journaled: failed target"] == 0 { |
| 959 | t.Fatalf("the target was stopped, or never released: %v", w.stats) |
| 960 | } |
| 961 | w.finish(t) |
| 962 | w.mustHold(t) |
| 963 | } |
| 964 | |
| 965 | // TestOperatorResumesARecordedFlush (operator-stop-unrecoverable-only; |
| 966 | // rev 10, Astra round 7 finding 2): the failed target completes its |
| 967 | // release sequence, records it and gets root's receipt (rev 11), then its |
| 968 | // agent restarts (the session's replicator is gone) before the release is |
| 969 | // journaled, and the deadline passes. The recorded flush is a release waiting to be |
| 970 | // journaled: the resolve resumes it, the target replays the record, and |
| 971 | // the nested relocation serves. The mutation stops the box. |
| 972 | func TestOperatorResumesARecordedFlush(t *testing.T) { |
| 973 | w := twoBoxWitness() |
| 974 | from, to := w.probeFailedTarget(t) |
| 975 | var r *releaseEntry |
| 976 | for i := 0; i < 60 && r == nil; i++ { |
| 977 | r = w.relocRelease(to, w.claim.clone()) // steps 1 to 3, the record and the receipt, not journaled |
| 978 | w.tick() |
| 979 | } |
| 980 | if r == nil { |
| 981 | t.Fatalf("the target's release sequence did not complete: %s", w.state()) |
| 982 | } |
| 983 | w.crash(to) |
| 984 | spare := w.boxByID(30) |
| 985 | if !w.reach(t, 2*releaseDeadline, "stuck at the release deadline", func() bool { return w.claim.Op.Stuck != "" }, w.act(from, spare), w.recoveryPass) { |
| 986 | w.mustHold(t) |
| 987 | } |
| 988 | w.resolveOnly() |
| 989 | if c := w.claim; to.Destroyed || c.Op.Stuck != "" || c.Op.Phase != "probe-failed" { |
| 990 | t.Fatalf("the resolve did not resume the recorded release (stopped %v): %s", to.Destroyed, w.state()) |
| 991 | } |
| 992 | w.servingOn(t, nil) |
| 993 | if w.stats["operator stopped a target that cannot release"] != 0 || w.stats["release: replayed from the recorded positions"] == 0 { |
| 994 | t.Fatalf("the target was stopped, or its record not replayed: %v", w.stats) |
| 995 | } |
| 996 | w.finish(t) |
| 997 | w.mustHold(t) |
| 998 | } |
| 999 | |
| 1000 | // TestStoppedTargetKeepsItsLocalCleanup (fence-discharge-qualified, |
| 1001 | // operator-stop-unrecoverable-only; rev 10, Astra round 7 findings 1 and |
| 1002 | // 2): the failed target loses its database and files with its agent, so |
| 1003 | // its release is unrecoverable; the operator stops the box and the next |
| 1004 | // resolve declares it lost and replays. The box's volume would be deleted |
| 1005 | // on termination, but it was stopped, not terminated: its disk is kept, |
| 1006 | // so when the recovery completes its park-local is still owed (claim §7.3 |
| 1007 | // D5, H19). The mutation of fence-discharge-qualified discharges it. |
| 1008 | func TestStoppedTargetKeepsItsLocalCleanup(t *testing.T) { |
| 1009 | w := twoBoxWitness() |
| 1010 | _, to := w.probeFailedTarget(t) |
| 1011 | to.DeleteOnTermination = true |
| 1012 | w.crash(to) |
| 1013 | to.Files, to.DB, to.receipt = nil, nil, nil |
| 1014 | if !w.reach(t, 3000, "serving after the operator's stop and replay", func() bool { |
| 1015 | c := w.claim |
| 1016 | return c.Op.Kind == "serving" && c.Op.Exec >= 0 && c.Op.Exec != to.ID |
| 1017 | }, w.actAll(), w.recoveryPass, w.wakeExec, w.resolveOnly) { |
| 1018 | w.mustHold(t) |
| 1019 | } |
| 1020 | if !to.Destroyed || to.Terminated || w.stats["operator stopped a target that cannot release"] == 0 { |
| 1021 | t.Fatalf("the target was not stopped by the operator: %s", w.state()) |
| 1022 | } |
| 1023 | if !w.owesLocal(w.claim, to) { |
| 1024 | t.Fatalf("the stopped target's local cleanup was discharged: %s", w.state()) |
| 1025 | } |
| 1026 | w.finish(t) // the operator's destroyed-local declaration, an explicit attestation, retires it |
| 1027 | w.mustHold(t) |
| 1028 | } |
| 1029 | |
| 1030 | // recordWithoutReceipt runs b's release sequence until its flushed |
| 1031 | // positions are recorded, and stops before the producers drain and root |
| 1032 | // persists the receipt. |
| 1033 | func (w *world) recordWithoutReceipt(t *testing.T, b *box) { |
| 1034 | for i := 0; i < 60 && b.relRecord == nil; i++ { |
| 1035 | w.relocRelease(b, w.claim.clone()) |
| 1036 | w.tick() |
| 1037 | } |
| 1038 | if r := b.relRecord; r == nil || r.Receipt { |
| 1039 | t.Fatalf("no record without a receipt on box %d: %+v %s", b.ID, r, w.state()) |
| 1040 | } |
| 1041 | } |
| 1042 | |
| 1043 | // TestReleaseRefusedAfterACrashBeforeTheReceipt (reloc-release-needs-receipt; |
| 1044 | // rev 11, Astra round 8): the source records its flushed positions, and |
| 1045 | // its agent crashes before root persists the stop receipt. The positions |
| 1046 | // are not proof of the stop: the restarted agent's release is refused |
| 1047 | // (RecoveryReleaseDrain), no release or pin appears, the deadline makes |
| 1048 | // the operation stuck, and the resolve returns the host to its source. |
| 1049 | // TestReleaseReplaysTheRecordedFlush is the other side: the crash after |
| 1050 | // the receipt, replayed. The mutation replays the positions alone. |
| 1051 | func TestReleaseRefusedAfterACrashBeforeTheReceipt(t *testing.T) { |
| 1052 | w := twoBoxWitness() |
| 1053 | from, to := w.boxes[0], w.boxes[3] |
| 1054 | w.serve(from, 1, app0) |
| 1055 | w.running(t, from) |
| 1056 | w.published(t, from) |
| 1057 | w.drainTo(t, to) |
| 1058 | w.appWrite(from) |
| 1059 | if !w.stepUntil(t, 1500, "releasing", w.atPhase("releasing")) { |
| 1060 | w.mustHold(t) |
| 1061 | } |
| 1062 | w.recordWithoutReceipt(t, from) |
| 1063 | w.crash(from) |
| 1064 | for i := 0; i < releaseDeadline+20; i++ { |
| 1065 | w.tick(w.act(from, to)) |
| 1066 | c := w.claim |
| 1067 | if c.Op.Release != nil || c.Op.Pin != nil { |
| 1068 | t.Fatalf("a release or a pin from recorded positions with no receipt (step %d): %s", i, w.state()) |
| 1069 | } |
| 1070 | if c.Op.Stuck != "" { |
| 1071 | break |
| 1072 | } |
| 1073 | } |
| 1074 | if c := w.claim; c.Op.Stuck == "" || c.Op.Phase != "releasing" { |
| 1075 | t.Fatalf("not stuck in releasing at the release deadline: %s", w.state()) |
| 1076 | } |
| 1077 | w.finish(t) |
| 1078 | w.mustHold(t) |
| 1079 | if c := w.claim; c.Op.Kind != "serving" || c.Op.Exec != from.ID { |
| 1080 | t.Fatalf("the operator's resolve did not return the host to its source: %s", w.state()) |
| 1081 | } |
| 1082 | } |
| 1083 | |
| 1084 | // TestOperatorStopsATargetWithNoReceipt (reloc-release-needs-receipt, |
| 1085 | // operator-stop-unrecoverable-only; rev 11, Astra round 8): the failed |
| 1086 | // target records its flushed positions and its agent restarts before root |
| 1087 | // persists the receipt. Its release can never come, so the operator stops |
| 1088 | // it, and the next resolve replays the operation; the host serves |
| 1089 | // elsewhere. The mutation treats the positions as recoverable and resumes |
| 1090 | // a release that replays them. |
| 1091 | func TestOperatorStopsATargetWithNoReceipt(t *testing.T) { |
| 1092 | w := twoBoxWitness() |
| 1093 | _, to := w.probeFailedTarget(t) |
| 1094 | w.recordWithoutReceipt(t, to) |
| 1095 | w.crash(to) |
| 1096 | if !w.reach(t, 3000, "serving after the operator's stop and replay", func() bool { |
| 1097 | c := w.claim |
| 1098 | return c.Op.Kind == "serving" && c.Op.Exec >= 0 && c.Op.Exec != to.ID |
| 1099 | }, w.actAll(), w.recoveryPass, w.wakeExec, w.resolveOnly) { |
| 1100 | w.mustHold(t) |
| 1101 | } |
| 1102 | if !to.Destroyed || w.stats["operator stopped a target that cannot release"] == 0 || w.stats["release journaled: failed target"] != 0 { |
| 1103 | t.Fatalf("the target was not stopped, or released from positions with no receipt: %v %s", w.stats, w.state()) |
| 1104 | } |
| 1105 | w.finish(t) |
| 1106 | w.mustHold(t) |
| 1107 | } |
| 1108 | |
| 1109 | // TestPinNeverShortOfTheRelease (reloc-pin-reaches-release): the listing |
| 1110 | // lags behind the journaled release (its newest file is not yet visible); |
| 1111 | // no pin is journaled short of the release's final_txid. |
| 1112 | func TestPinNeverShortOfTheRelease(t *testing.T) { |
| 1113 | w := twoBoxWitness() |
| 1114 | from, to := w.boxes[0], w.boxes[3] |
| 1115 | w.serve(from, 1, app0) |
| 1116 | w.running(t, from) |
| 1117 | w.published(t, from) |
| 1118 | w.appWrite(from) |
| 1119 | w.drainTo(t, to) |
| 1120 | if !w.stepUntil(t, 1500, "the release journaled", func() bool { c := w.claim; return c.Op.Release != nil && c.Op.Pin == nil }) { |
| 1121 | w.mustHold(t) |
| 1122 | } |
| 1123 | c := w.claim |
| 1124 | bk := w.buckets[0] |
| 1125 | var hidden string |
| 1126 | var vs []*version |
| 1127 | for _, k := range bk.keys(dataKey(c.Holder.Location, c.Op.Release.Gen, "")) { |
| 1128 | if l, _, max, ok := ltxLevel(k); ok && l == 0 && max == c.Op.Release.FinalTX { |
| 1129 | hidden, vs = k, bk.objs[k] |
| 1130 | delete(bk.objs, k) |
| 1131 | } |
| 1132 | } |
| 1133 | if hidden == "" { |
| 1134 | t.Fatalf("no level-0 file at the release's final_txid %d", c.Op.Release.FinalTX) |
| 1135 | } |
| 1136 | w.drive(20, never, w.act(from, to)) |
| 1137 | if p := w.claim.Op.Pin; p != nil && p.TX < c.Op.Release.FinalTX { |
| 1138 | t.Fatalf("pinned at TXID %d, short of the release's %d", p.TX, c.Op.Release.FinalTX) |
| 1139 | } |
| 1140 | bk.objs[hidden] = vs |
| 1141 | w.servingOn(t, to) |
| 1142 | w.finish(t) |
| 1143 | w.mustHold(t) |
| 1144 | } |
| 1145 | |
| 1146 | // ---------- finding 8: claim §11's additional witnesses ---------- |
| 1147 | |
| 1148 | // TestCrashAtEachOpenBoundary (claim §11 "Crash between prepared |
| 1149 | // admission, marker publication and replication opening"): the target's |
| 1150 | // agent crashes exactly at each boundary of P's first admitted session: |
| 1151 | // admitted with the app starting (its launch environment held unready), |
| 1152 | // admitted with generation.json's PUT in flight (held), generation.json |
| 1153 | // published and replication not yet open (rev 5, Astra round 2: the |
| 1154 | // witness yield openYield stops the start between the two), and |
| 1155 | // replication open; in a one-box and a two-box region. |
| 1156 | func TestCrashAtEachOpenBoundary(t *testing.T) { |
| 1157 | for _, stage := range []string{"app", "marker", "published", "open"} { |
| 1158 | t.Run(stage, func(t *testing.T) { |
| 1159 | bothShapes(t, func(t *testing.T, w *world, boxes int) { |
| 1160 | from := w.serveAndPublish(t) |
| 1161 | w.timerOn(from) |
| 1162 | loc := w.claim.Holder.Location |
| 1163 | if boxes == 2 { |
| 1164 | w.drainTo(t, nil) |
| 1165 | } else { |
| 1166 | w.terminate(from) |
| 1167 | } |
| 1168 | if stage == "marker" { |
| 1169 | w.holdKey(genMarkerKey(loc)) |
| 1170 | } |
| 1171 | yielded := false |
| 1172 | if stage == "published" { |
| 1173 | w.openYield = func(b *box) bool { |
| 1174 | if yielded || b.open == nil || !b.open.Prepared { |
| 1175 | return false |
| 1176 | } |
| 1177 | yielded = true |
| 1178 | return true |
| 1179 | } |
| 1180 | } |
| 1181 | var to *box |
| 1182 | hit := w.stepUntil(t, 2000, "P's open at "+stage, func() bool { |
| 1183 | c := w.claim |
| 1184 | if c.Op.Kind != "relocating" || c.Op.Tgt < 0 { |
| 1185 | return false |
| 1186 | } |
| 1187 | to = w.boxByID(c.Op.Tgt) |
| 1188 | if stage == "app" && c.Op.Phase == "probing" && to.envUnreadyFor == 0 { |
| 1189 | to.envUnreadyFor = 20 // the launch environment is not ready yet |
| 1190 | } |
| 1191 | if stage == "open" { |
| 1192 | return preparedOpened(c) && to.open == nil && hasMain(to) |
| 1193 | } |
| 1194 | if stage == "published" { |
| 1195 | if !yielded || to.open == nil || to.open.stage != "open" { |
| 1196 | return false |
| 1197 | } |
| 1198 | if g, _ := w.bucketGen(to, loc); g != to.open.Gen { |
| 1199 | t.Fatalf("yielded before generation.json named %q", to.open.Gen) |
| 1200 | } |
| 1201 | return true |
| 1202 | } |
| 1203 | return to.open != nil && to.open.Prepared && to.open.stage == stage |
| 1204 | }) |
| 1205 | if !hit { |
| 1206 | return |
| 1207 | } |
| 1208 | w.crash(to) |
| 1209 | to.envUnreadyFor = 0 |
| 1210 | w.hold, w.openYield = nil, nil |
| 1211 | w.servingOn(t, nil) |
| 1212 | w.finish(t) |
| 1213 | }) |
| 1214 | }) |
| 1215 | } |
| 1216 | } |
| 1217 | |
| 1218 | // TestAutomaticTargetLossRacesRecordCommit (claim §11 "Automatic target |
| 1219 | // loss racing the record commit CAS"): a move's target is terminated |
| 1220 | // just before, and just after, its record commit CAS; the regional |
| 1221 | // worker's automatic fence takes the operator's record-first path, so the |
| 1222 | // claim follows whichever the record holds: a rollback to the source |
| 1223 | // before the commit, a committed landing (then a failover in the target's |
| 1224 | // region) after it. |
| 1225 | func TestAutomaticTargetLossRacesRecordCommit(t *testing.T) { |
| 1226 | for _, when := range []string{"before the commit", "after the commit"} { |
| 1227 | t.Run(when, func(t *testing.T) { |
| 1228 | w := twoBoxWitness() |
| 1229 | src := w.boxes[0] |
| 1230 | w.serve(src, 1, app0) |
| 1231 | w.running(t, src) |
| 1232 | w.published(t, src) |
| 1233 | w.moveTo(1) |
| 1234 | if w.claim.Op.Kind != "moving" { |
| 1235 | t.Fatalf("no move: %s", w.state()) |
| 1236 | } |
| 1237 | tgt := w.boxByID(w.claim.Op.Tgt) |
| 1238 | ok := w.stepUntil(t, 1500, "the commit boundary", func() bool { |
| 1239 | c := w.claim |
| 1240 | if when == "before the commit" { |
| 1241 | return c.Op.Kind == "landing" && c.Op.LandPhase == "probed" && w.rec != nil && w.rec.MoveState == "landing" && w.now >= c.Op.ProbedAt+15 |
| 1242 | } |
| 1243 | return w.rec != nil && w.rec.MoveState == "committed" && c.Op.Kind == "landing" && !c.Op.Committed |
| 1244 | }) |
| 1245 | if !ok { |
| 1246 | w.mustHold(t) |
| 1247 | } |
| 1248 | landing := w.claim.Op.ID |
| 1249 | w.terminate(tgt) |
| 1250 | if when == "after the commit" { |
| 1251 | // the failover keeps the landing's op id, as hostclaim |
| 1252 | // ConvertCommittedLanding does: the move's admission |
| 1253 | // registration is drained unless the claim carries it |
| 1254 | if !w.stepUntil(t, 400, "the committed landing converted", func() bool { return w.claim.Op.Kind == "relocating" }) { |
| 1255 | w.mustHold(t) |
| 1256 | } |
| 1257 | if w.claim.Op.ID != landing { |
| 1258 | t.Fatalf("the committed landing %s converted to relocation %s, want the landing's op id kept: %s", landing, w.claim.Op.ID, w.state()) |
| 1259 | } |
| 1260 | } |
| 1261 | w.servingOn(t, nil) |
| 1262 | w.finish(t) |
| 1263 | w.mustHold(t) |
| 1264 | c := w.claim |
| 1265 | if when == "before the commit" && (w.rec.Region != src.Region || w.execBox(c).Region != src.Region) { |
| 1266 | t.Fatalf("a loss before the commit did not return the host to the source's region: %s", w.state()) |
| 1267 | } |
| 1268 | if when == "after the commit" && (w.rec.Region != 1 || w.execBox(c).Region != 1) { |
| 1269 | t.Fatalf("a loss after the commit did not keep the host in the target's region: %s", w.state()) |
| 1270 | } |
| 1271 | }) |
| 1272 | } |
| 1273 | } |
| 1274 | |
| 1275 | // ---------- R18-2: the tie-break follows the host ---------- |
| 1276 | |
| 1277 | // TestTieHashFollowsTheHost (tie-hash-host; 1b-next R18-2): on equal |
| 1278 | // loads two hostnames may choose different boxes; the model's tie-break |
| 1279 | // hashes each host's own name. |
| 1280 | func TestTieHashFollowsTheHost(t *testing.T) { |
| 1281 | w := regionWorld(2) |
| 1282 | a, b := w.boxes[0], w.peer() |
| 1283 | sum := func(host string, id int) uint32 { // claim P5-D2, computed here independently of the model |
| 1284 | s := sha256.Sum256([]byte(fmt.Sprintf("%s\x00%d", host, id))) |
| 1285 | return uint32(s[0])<<24 | uint32(s[1])<<16 | uint32(s[2])<<8 | uint32(s[3]) |
| 1286 | } |
| 1287 | wantH, wantH2 := a, a |
| 1288 | if sum("h", b.ID) > sum("h", a.ID) { |
| 1289 | wantH = b |
| 1290 | } |
| 1291 | if sum("h2", b.ID) > sum("h2", a.ID) { |
| 1292 | wantH2 = b |
| 1293 | } |
| 1294 | if wantH == wantH2 { |
| 1295 | t.Skip("hosts h and h2 choose the same box among boxes 0 and 3") |
| 1296 | } |
| 1297 | if got := w.chooseBoxFor("h", 0, app0); got != wantH { |
| 1298 | t.Fatalf("host h chose %v, want %d", got, wantH.ID) |
| 1299 | } |
| 1300 | if got := w.chooseBoxFor("h2", 0, app0); got != wantH2 { |
| 1301 | t.Fatalf("host h2 chose %v, want %d", got, wantH2.ID) |
| 1302 | } |
| 1303 | } |
| 1304 | |
| 1305 | // ---------- the final review's minor 4: a released source stays eligible ---------- |
| 1306 | |
| 1307 | // TestOperatorReplayKeepsAReleasedSourceEligible (model ruling 13; final |
| 1308 | // review minor 4): a drain's source journals its release in releasing, |
| 1309 | // still op.exec, and the target is then terminated with no spare (the |
| 1310 | // group launches none). The dead target's bound makes the operation |
| 1311 | // stuck with the released source as op.exec. The operator's replay |
| 1312 | // makes the source eligible again: it excludes only a failed target, |
| 1313 | // never the source, so the host serves on the source. Excluding the |
| 1314 | // released source left the replay waiting at pinning for a spare that |
| 1315 | // never comes. |
| 1316 | func TestOperatorReplayKeepsAReleasedSourceEligible(t *testing.T) { |
| 1317 | w := twoBoxWitness() |
| 1318 | from, to := w.boxes[0], w.boxes[3] |
| 1319 | w.serve(from, 1, app0) |
| 1320 | w.running(t, from) |
| 1321 | w.published(t, from) |
| 1322 | w.drainTo(t, to) |
| 1323 | w.oneBoxRegion[0] = true // no replacement launches: no spare |
| 1324 | if !w.stepUntil(t, 400, "the source's release journaled", func() bool { |
| 1325 | c := w.claim |
| 1326 | return c.Op.Release != nil && c.Op.Release.Box == from.ID |
| 1327 | }) { |
| 1328 | w.mustHold(t) |
| 1329 | } |
| 1330 | if c := w.claim; c.Op.Phase != "releasing" || c.Op.Exec != from.ID { |
| 1331 | t.Fatalf("not released in releasing with the source op.exec: %s", w.state()) |
| 1332 | } |
| 1333 | w.terminate(to) |
| 1334 | if !w.reach(t, 3*releaseDeadline, "stuck", func() bool { return w.claim.Op.Stuck != "" }, w.act(from), w.recoveryPass) { |
| 1335 | w.mustHold(t) |
| 1336 | } |
| 1337 | if c := w.claim; c.Op.Exec != from.ID || c.Op.Src != from.ID || c.Op.Release == nil || c.Op.Release.Box != from.ID { |
| 1338 | t.Fatalf("not stuck with the released source as op.exec: %s", w.state()) |
| 1339 | } |
| 1340 | w.resolveOnly() |
| 1341 | if c := w.claim; c.Op.Phase != "pinning" || c.Op.Stuck != "" || !c.Op.Resolved { |
| 1342 | t.Fatalf("the resolve did not replay from pinning: %s", w.state()) |
| 1343 | } |
| 1344 | if c := w.claim; contains(c.Op.Excluded, from.ID) { |
| 1345 | t.Fatalf("the replay excludes the released source %d: excluded=%v", from.ID, c.Op.Excluded) |
| 1346 | } |
| 1347 | w.servingOn(t, from) |
| 1348 | w.finish(t) |
| 1349 | w.mustHold(t) |
| 1350 | } |
| 1351 | |
amadan