| 1 | package hostlife |
| 2 | |
| 3 | import "testing" |
| 4 | |
| 5 | // ---------- L18 joint round findings 2 to 4: witnesses and direct tests ---------- |
| 6 | |
| 7 | // completedDrain drains the host from box 0 to box 3 with fault set on |
| 8 | // the target first, and drives until the claim serves on box 3 with its |
| 9 | // completion recorded (regionha_completion_test.go). |
| 10 | func (w *world) completedDrain(t *testing.T, fault func(to *box)) (from, to *box, ok bool) { |
| 11 | w.twoBox() |
| 12 | from, to = w.boxes[0], w.boxes[3] |
| 13 | w.serve(from, 1, app0) |
| 14 | w.running(t, from) |
| 15 | if fault != nil { |
| 16 | fault(to) |
| 17 | } |
| 18 | w.drainTo(t, to) |
| 19 | w.extraBox(0, 30) // a target for a successor drain from box 3 (box 0 owes its park-local) |
| 20 | ok = w.reach(t, 1500, "serving on box 3 with its completion recorded", func() bool { |
| 21 | c := w.claim |
| 22 | return c.Op.Kind == "serving" && c.Op.Exec == to.ID && c.Op.Completed != nil |
| 23 | }, w.act(from, to), w.recoveryPass) |
| 24 | return from, to, ok |
| 25 | } |
| 26 | |
| 27 | // marked: the serving op's completion is marked discharged. |
| 28 | func (w *world) marked() bool { |
| 29 | d := w.claim.Op.Completed |
| 30 | return w.claim.Op.Kind == "serving" && d != nil && d.Discharged |
| 31 | } |
| 32 | |
| 33 | // wProbeStopBeforeServing (reloc-probe-stop): a drain completes; the |
| 34 | // probe's session is stopped with root's receipt before the serving CAS, |
| 35 | // and the host wakes on box 3 afterwards. Completed over the running |
| 36 | // probe, its session outlives the kind change on a box whose |
| 37 | // participation is not discharged (INV-31). |
| 38 | func wProbeStopBeforeServing(t *testing.T, w *world) { |
| 39 | from, to, ok := w.completedDrain(t, nil) |
| 40 | if !ok { |
| 41 | return |
| 42 | } |
| 43 | _ = from |
| 44 | w.reach(t, 200, "the host runs on box 3 after the completion", func() bool { return hasMain(to) && w.marked() }, w.actAll(), w.recoveryPass, func() { w.wake(to) }) |
| 45 | w.finish(t) |
| 46 | } |
| 47 | |
| 48 | // wProbeStopMeetsAnotherFence (reloc-probe-fence-owner, the stop): the |
| 49 | // probe has passed when another operation's fence closes the host on box |
| 50 | // 3. The probe stop refuses with nothing stopped, and ruling 67 makes the |
| 51 | // relocation stuck for an operator. Installed over that fence, the stop |
| 52 | // later lifts it as its own, and the serving wake starts the host under a |
| 53 | // fence its owner never lifted (INV-32). |
| 54 | func wProbeStopMeetsAnotherFence(t *testing.T, w *world) { |
| 55 | w.twoBox() |
| 56 | from, to := w.boxes[0], w.boxes[3] |
| 57 | w.serve(from, 1, app0) |
| 58 | w.running(t, from) |
| 59 | w.drainTo(t, to) |
| 60 | if !w.reach(t, 1500, "the probe passed", func() bool { |
| 61 | c := w.claim |
| 62 | return c.Op.Kind == "relocating" && c.Op.Phase == "probed" && c.Op.ProbeStop == nil |
| 63 | }, w.act(from, to), w.recoveryPass) { |
| 64 | return |
| 65 | } |
| 66 | to.actFence, to.foreignFence = "op-elsewhere", "op-elsewhere" |
| 67 | w.reach(t, 3*releaseDeadline, "stuck with the probe unstopped", func() bool { return w.claim.Op.Stuck != "" }, w.act(from, to), w.recoveryPass, func() { w.wake(to) }) |
| 68 | } |
| 69 | |
| 70 | // wLiftAfterRestartMeetsAnotherFence (reloc-probe-fence-owner, the lift, |
| 71 | // a crash): the completion's discharge fails for a while; the target's |
| 72 | // agent restarts, which empties the activator's fences, and another |
| 73 | // operation fences the host. The completion pass then discharges, |
| 74 | // releases the hold and lifts only its own fence: the other one stays |
| 75 | // and the serving wake waits. Lifting any fence opens the host under it |
| 76 | // (INV-32). |
| 77 | func wLiftAfterRestartMeetsAnotherFence(t *testing.T, w *world) { |
| 78 | _, to, ok := w.completedDrain(t, func(to *box) { to.dischargeFails = true }) |
| 79 | if !ok { |
| 80 | return |
| 81 | } |
| 82 | w.crash(to) |
| 83 | to.actFence, to.foreignFence = "op-elsewhere", "op-elsewhere" |
| 84 | to.dischargeFails = false |
| 85 | w.reach(t, 100, "the completion marked", w.marked, w.act(to), func() { w.wake(to) }) |
| 86 | w.drive(20, never, w.act(to), func() { w.wake(to) }) |
| 87 | } |
| 88 | |
| 89 | // wDischargeFailsHoldsTheRow (reloc-completion-discharge-first, refusal |
| 90 | // and crash): C10's discharge fails for thirty steps; the row stays held |
| 91 | // and the fence installed, so no serving wake moves the holder off the |
| 92 | // stopped probe session. Then the discharge succeeds, the agent crashes |
| 93 | // after the hold's release, and the next pass resumes from the claim: |
| 94 | // discharged again on positive evidence, marked, and the host serves. |
| 95 | // With the hold released and the fence lifted first, a request admits a |
| 96 | // writer before the discharge (INV-31). |
| 97 | func wDischargeFailsHoldsTheRow(t *testing.T, w *world) { |
| 98 | _, to, ok := w.completedDrain(t, func(to *box) { to.dischargeFails = true }) |
| 99 | if !ok { |
| 100 | return |
| 101 | } |
| 102 | w.drive(30, never, w.act(to), w.recoveryPass, func() { w.wake(to) }) |
| 103 | to.dischargeFails, to.crashAfter = false, "release" |
| 104 | w.reach(t, 200, "the completion marked after a crash", w.marked, w.act(to), w.recoveryPass, func() { w.wake(to) }) |
| 105 | w.finish(t) |
| 106 | } |
| 107 | |
| 108 | // wMarkLastAfterAFailedDischarge (reloc-completion-mark-last): the |
| 109 | // discharge fails, and a drain from box 3 is asked for meanwhile; with |
| 110 | // the mark last the record stays unmarked and the barrier refuses it |
| 111 | // until the discharge succeeds. Marked first, the barrier lets the drain |
| 112 | // name box 3 while its participation is still the completed op's |
| 113 | // (INV-31), and the discharge is never retried. |
| 114 | func wMarkLastAfterAFailedDischarge(t *testing.T, w *world) { |
| 115 | _, to, ok := w.completedDrain(t, func(to *box) { to.dischargeFails = true }) |
| 116 | if !ok { |
| 117 | return |
| 118 | } |
| 119 | w.drive(10, never, w.act(to), w.recoveryPass) |
| 120 | w.userRelocate() |
| 121 | w.drive(5, never, w.act(to), w.recoveryPass) |
| 122 | to.dischargeFails = false |
| 123 | if !w.reach(t, 200, "the completion marked", w.marked, w.act(to), w.recoveryPass, func() { w.wake(to) }) { |
| 124 | return |
| 125 | } |
| 126 | w.running(t, to) |
| 127 | w.userRelocate() |
| 128 | w.servingOn(t, nil) |
| 129 | w.finish(t) |
| 130 | } |
| 131 | |
| 132 | // wFailedMarkRetriedFromTheClaim (reloc-completion-from-claim, a failed |
| 133 | // mark and a crash): the mark fails once after the hold's release and |
| 134 | // the lift; the next pass finds the unmarked completion from the claim |
| 135 | // record and marks it. A second drain completes with a crash after the |
| 136 | // lift, and is marked on the next pass too. Found from the row's hold, |
| 137 | // which the release removed, the completion is never marked, and nothing |
| 138 | // starts from the host again (INV-3). |
| 139 | func wFailedMarkRetriedFromTheClaim(t *testing.T, w *world) { |
| 140 | _, to, ok := w.completedDrain(t, func(to *box) { to.markFails = true }) |
| 141 | if !ok { |
| 142 | return |
| 143 | } |
| 144 | if !w.reach(t, 200, "the completion marked after a failed mark", w.marked, w.act(to), w.recoveryPass, func() { w.wake(to) }) { |
| 145 | return |
| 146 | } |
| 147 | w.running(t, to) |
| 148 | w.userRelocate() |
| 149 | back := w.boxByID(w.claim.Op.Tgt) |
| 150 | if back == nil || w.claim.Op.Kind != "relocating" { |
| 151 | return |
| 152 | } |
| 153 | back.crashAfter = "lift" |
| 154 | w.servingOn(t, back) |
| 155 | w.reach(t, 200, "the second completion marked after a crash", w.marked, w.actAll(), w.recoveryPass, w.wakeExec) |
| 156 | w.finish(t) |
| 157 | } |
| 158 | |
| 159 | // wSuccessorWaitsForTheDischarge (reloc-completion-barrier): while the |
| 160 | // completion's discharge fails, a drain, a move and a push from box 3 |
| 161 | // are all refused; once it is discharged and marked the drain starts and |
| 162 | // completes. Without the barrier the drain names box 3 while its |
| 163 | // participation is still the completed op's (INV-31). |
| 164 | func wSuccessorWaitsForTheDischarge(t *testing.T, w *world) { |
| 165 | _, to, ok := w.completedDrain(t, func(to *box) { to.dischargeFails = true }) |
| 166 | if !ok { |
| 167 | return |
| 168 | } |
| 169 | w.userRelocate() |
| 170 | w.moveTo(1) |
| 171 | w.userPush() |
| 172 | w.drive(5, never, w.act(to), w.recoveryPass) |
| 173 | if w.claim.Op.Kind != "serving" { |
| 174 | return // the mutation started one; INV-31 was recorded at the step |
| 175 | } |
| 176 | to.dischargeFails = false |
| 177 | if !w.reach(t, 200, "the completion marked", w.marked, w.act(to), w.recoveryPass, func() { w.wake(to) }) { |
| 178 | return |
| 179 | } |
| 180 | w.running(t, to) |
| 181 | w.userRelocate() |
| 182 | w.servingOn(t, nil) |
| 183 | w.finish(t) |
| 184 | } |
| 185 | |
| 186 | // wCompletionWithRegionHAOff (reloc-completion-claim-mode, ruling 82, |
| 187 | // a restart): the discharge fails, then region HA is turned off and the |
| 188 | // target's agent restarts; the completion-only pass still discharges, |
| 189 | // releases, lifts and marks, and the host serves on box 3. Run only with |
| 190 | // region HA on, the completion is never finished (INV-3). |
| 191 | func wCompletionWithRegionHAOff(t *testing.T, w *world) { |
| 192 | _, to, ok := w.completedDrain(t, func(to *box) { to.dischargeFails = true }) |
| 193 | if !ok { |
| 194 | return |
| 195 | } |
| 196 | w.haOff = true |
| 197 | w.crash(to) |
| 198 | to.dischargeFails = false |
| 199 | w.reach(t, 200, "the completion finished with region HA off", func() bool { return w.marked() && hasMain(to) }, w.act(to), w.recoveryPass, func() { w.wake(to) }) |
| 200 | w.finish(t) |
| 201 | } |
| 202 | |
| 203 | // wServingWakeHandsOver (reloc-completed-handover, N6): after the |
| 204 | // completion the first serving wake on box 3 hands over from the probe's |
| 205 | // receipt and is cold, as claimServingOwners.Select makes every serving |
| 206 | // wake (L18 joint round 2 finding 1): a fresh generation, a new |
| 207 | // installation stamp, a writes_lost record for the stopped probe session, |
| 208 | // and P retired by the park-generation that follows. Routed to main's |
| 209 | // stop, its serving-shaped fence replaces the probe's and the handover |
| 210 | // refuses for good (INV-3). |
| 211 | func wServingWakeHandsOver(t *testing.T, w *world) { |
| 212 | _, to, ok := w.completedDrain(t, nil) |
| 213 | if !ok { |
| 214 | return |
| 215 | } |
| 216 | d, p, stamp := *w.claim.Op.Completed, w.claim.Holder.Gen, w.claim.Holder.Stamp |
| 217 | if !w.reach(t, 200, "the host runs on box 3", func() bool { return hasMain(to) && to.open == nil }, w.actAll(), w.recoveryPass, func() { w.wake(to) }) { |
| 218 | return |
| 219 | } |
| 220 | w.checkFirstWakeCold(t, d, p, stamp) |
| 221 | w.reach(t, 600, "P retired by its park-generation", func() bool { return !w.hasFilesAnywhere(p) && len(w.claim.Obl) == 0 }, w.actAll(), w.recoveryPass) |
| 222 | w.finish(t) |
| 223 | } |
| 224 | |
| 225 | // checkFirstWakeCold asserts the first serving wake after completion d of |
| 226 | // prepared generation p at stamp: cold, into a fresh generation, at a new |
| 227 | // stamp, declaring the stopped probe session's writes_lost. |
| 228 | func (w *world) checkFirstWakeCold(t *testing.T, d completedEntry, p string, stamp int) { |
| 229 | t.Helper() |
| 230 | h := w.claim.Holder |
| 231 | l := lastAdmission(h) |
| 232 | switch { |
| 233 | case h.Gen == p || h.Gen == "": |
| 234 | t.Errorf("the first serving wake continued P %q (generation %q)", p, h.Gen) |
| 235 | case h.Stamp == stamp: |
| 236 | t.Errorf("the first serving wake kept the installation stamp %d", stamp) |
| 237 | case l == nil || l.Mode != "cold" || l.Box != d.Box || l.Gen != h.Gen: |
| 238 | t.Errorf("the first serving wake was not a cold admission on box %d: %+v", d.Box, l) |
| 239 | case l.Lost == nil || l.Lost.Session != d.Session || l.Lost.Box != d.Box || l.Lost.Gen != p: |
| 240 | t.Errorf("the first serving wake did not declare the stopped probe session %d's writes_lost: %+v", d.Session, l.Lost) |
| 241 | } |
| 242 | } |
| 243 | |
| 244 | // hasFilesAnywhere: some region's bucket holds an object of generation g |
| 245 | // of the holder's location. |
| 246 | func (w *world) hasFilesAnywhere(g string) bool { |
| 247 | for _, b := range w.boxes { |
| 248 | if w.hasFiles(b, w.claim.Holder.Location, g) { |
| 249 | return true |
| 250 | } |
| 251 | } |
| 252 | return false |
| 253 | } |
| 254 | |
| 255 | // probeWriteThenServe drains box 0 to box 3; once the probe passed, the |
| 256 | // probe session acknowledges a write its replicator has not shipped; then |
| 257 | // the relocation completes and the first serving wake on box 3 runs. It |
| 258 | // returns the write, the completion and the P and stamp it completed with. |
| 259 | func (w *world) probeWriteThenServe(t *testing.T) (wr int, d completedEntry, p string, stamp int, ok bool) { |
| 260 | w.twoBox() |
| 261 | from, to := w.boxes[0], w.boxes[3] |
| 262 | w.serve(from, 1, app0) |
| 263 | w.running(t, from) |
| 264 | w.drainTo(t, to) |
| 265 | if !w.reach(t, 1500, "the probe passed", func() bool { c := w.claim; return c.Op.Phase == "probed" && c.Op.ProbeStop == nil && hasMain(to) }, w.act(from, to), w.recoveryPass) { |
| 266 | return 0, d, "", 0, false |
| 267 | } |
| 268 | wr = w.appWrite(to) // acknowledged by the probe session, not yet replicated |
| 269 | if !w.reach(t, 200, "serving on box 3", func() bool { return w.claim.Op.Kind == "serving" && w.claim.Op.Completed != nil }, w.act(from, to), w.recoveryPass) { |
| 270 | return 0, d, "", 0, false |
| 271 | } |
| 272 | d, p, stamp = *w.claim.Op.Completed, w.claim.Holder.Gen, w.claim.Holder.Stamp |
| 273 | ok = w.reach(t, 200, "the host runs on box 3", func() bool { return hasMain(to) && to.open == nil }, w.actAll(), w.recoveryPass, func() { w.wake(to) }) |
| 274 | return wr, d, p, stamp, ok |
| 275 | } |
| 276 | |
| 277 | // wProbeStopFlushes (reloc-probe-stop-flush, ruling 87): the probe |
| 278 | // session acknowledges a write the replicator has not shipped; the probe |
| 279 | // stop flushes it before root's stop, so the first serving wake, cold, |
| 280 | // declares a writes_lost that names none of the probe session's writes. |
| 281 | // Stopped with nothing flushed, that wake declares the write lost |
| 282 | // (INV-34). |
| 283 | func wProbeStopFlushes(t *testing.T, w *world) { |
| 284 | if _, _, _, _, ok := w.probeWriteThenServe(t); !ok { |
| 285 | return |
| 286 | } |
| 287 | w.finish(t) |
| 288 | } |
| 289 | |
| 290 | // TestFirstServingWakeIsCold (L18 joint round 2 finding 1, ruling 87): |
| 291 | // the first serving wake after the completion is cold (a fresh |
| 292 | // generation, a new stamp, a writes_lost for the stopped probe session), |
| 293 | // and since the probe stop flushed, the write the probe session |
| 294 | // acknowledged is in the new generation and no write is declared lost. |
| 295 | func TestFirstServingWakeIsCold(t *testing.T) { |
| 296 | w := newWitness() |
| 297 | wr, d, p, stamp, ok := w.probeWriteThenServe(t) |
| 298 | if !ok { |
| 299 | w.mustHold(t) |
| 300 | } |
| 301 | w.mustHold(t) |
| 302 | w.checkFirstWakeCold(t, d, p, stamp) |
| 303 | to := w.boxByID(d.Box) |
| 304 | for _, a := range w.acks { |
| 305 | if a.W == wr && a.Exempt { |
| 306 | t.Fatalf("the probe session's write %d was declared lost: %s", wr, w.state()) |
| 307 | } |
| 308 | } |
| 309 | if to.DB == nil || !contains(to.DB.Log, wr) || w.stats["writes declared lost"] != 0 { |
| 310 | t.Fatalf("the probe session's write %d is not in the first serving wake's database (%d writes declared lost): %s", wr, w.stats["writes declared lost"], w.state()) |
| 311 | } |
| 312 | w.finish(t) |
| 313 | w.mustHold(t) |
| 314 | } |
| 315 | |
| 316 | // TestProbeStopRefusedAfterARestartBeforeTheRecord (reloc-probe-stop-flush, |
| 317 | // ruling 87): the probe passed, its flush is held, and the target's agent |
| 318 | // restarts before the positions are recorded. The new agent has no |
| 319 | // replicator for the probe session, so the probe stop refuses; nothing is |
| 320 | // journaled, and ruling 67's deadline makes the relocation stuck, as a |
| 321 | // release would be. The mutation stops at once, and this test fails. |
| 322 | func TestProbeStopRefusedAfterARestartBeforeTheRecord(t *testing.T) { |
| 323 | w := twoBoxWitness() |
| 324 | from, to := w.boxes[0], w.boxes[3] |
| 325 | w.serve(from, 1, app0) |
| 326 | w.running(t, from) |
| 327 | w.drainTo(t, to) |
| 328 | w.reach(t, 1500, "the probe passed", func() bool { c := w.claim; return c.Op.Phase == "probed" && c.Op.ProbeStop == nil && hasMain(to) }, w.act(from, to), w.recoveryPass) |
| 329 | w.appWrite(to) |
| 330 | loc := w.claim.Holder.Location |
| 331 | w.holdKey(loc) |
| 332 | w.drive(3, never, w.act(from, to)) |
| 333 | if to.probeRecord != nil || w.claim.Op.ProbeStop != nil { |
| 334 | t.Fatalf("the probe stop recorded or journaled with its flush held: %s", w.state()) |
| 335 | } |
| 336 | w.crash(to) |
| 337 | w.hold = nil |
| 338 | w.completeHeld(loc) |
| 339 | w.reach(t, 3*releaseDeadline, "stuck at the probe stop's deadline", func() bool { return w.claim.Op.Stuck != "" }, w.act(from, to), w.recoveryPass) |
| 340 | w.mustHold(t) |
| 341 | if c := w.claim; c.Op.Phase != "probed" || c.Op.ProbeStop != nil || to.probeRecord != nil || w.stats["probe stop: no replicator or files directory and no record, refused"] == 0 { |
| 342 | t.Fatalf("the probe stop did not refuse until the deadline: %s", w.state()) |
| 343 | } |
| 344 | } |
| 345 | |
| 346 | // TestNoAdmissionBetweenStopAndFence (reloc-admits-probe-only, L18 joint |
| 347 | // round 2 finding 3): the source's release stops its writers and closes |
| 348 | // the session's launch journal, and its flush is held, so root's |
| 349 | // admission fence (Collect) has not come. A request reaches the source, |
| 350 | // then its agent restarts and another request reaches it: no writer is |
| 351 | // admitted in the interval, because relocatingAdmission admits only the |
| 352 | // probe. The mutation admits the source's wake, and this test fails. |
| 353 | func TestNoAdmissionBetweenStopAndFence(t *testing.T) { |
| 354 | w := twoBoxWitness() |
| 355 | from, to := w.boxes[0], w.boxes[3] |
| 356 | w.serve(from, 1, app0) |
| 357 | w.running(t, from) |
| 358 | w.appWrite(from) |
| 359 | loc := w.claim.Holder.Location |
| 360 | w.holdKey(loc) // the flush is held: no positions, no Collect |
| 361 | w.drainTo(t, to) |
| 362 | w.reach(t, 200, "the source stopped, its launch journal closed", func() bool { |
| 363 | return w.claim.Op.Phase == "releasing" && from.launchClosedOp == w.claim.Op.ID && !hasMain(from) |
| 364 | }, w.act(from, to), w.recoveryPass) |
| 365 | session, op := w.claim.Holder.Session, w.claim.Op.ID |
| 366 | for _, restart := range []bool{false, true} { |
| 367 | if restart { |
| 368 | w.crash(from) |
| 369 | } |
| 370 | w.drive(5, never, func() { w.wake(from) }, w.act(from, to)) |
| 371 | if from.relFence == op { |
| 372 | t.Fatalf("root's admission fence came before the flush") |
| 373 | } |
| 374 | if hasMain(from) || w.claim.Holder.Session != session || from.open != nil { |
| 375 | t.Fatalf("a wake admitted a writer between the stop and the fence (restart %v): %s", restart, w.state()) |
| 376 | } |
| 377 | } |
| 378 | if w.stats["admission refused: relocating admits only the probe"] == 0 { |
| 379 | t.Fatalf("no wake reached the admission: %v", w.stats) |
| 380 | } |
| 381 | w.mustHold(t) |
| 382 | w.hold = nil |
| 383 | w.completeHeld(loc) |
| 384 | w.servingOn(t, nil) |
| 385 | w.finish(t) |
| 386 | w.mustHold(t) |
| 387 | } |
| 388 | |
| 389 | // wDelayedSourceHoldsItsRow (reloc-completed-catchup-hold, finding 3): |
| 390 | // the source releases and its admission fence closes, then its |
| 391 | // reconcile is delayed until the target has completed. Its first |
| 392 | // reconcile after that holds the row for the completed op, never |
| 393 | // leaving it ready, until its park-local retires it. Without the |
| 394 | // catch-up hold the row its release left ready stays ready (INV-33). |
| 395 | func wDelayedSourceHoldsItsRow(t *testing.T, w *world) { |
| 396 | w.twoBox() |
| 397 | from, to := w.boxes[0], w.boxes[3] |
| 398 | w.serve(from, 1, app0) |
| 399 | w.running(t, from) |
| 400 | w.drainTo(t, to) |
| 401 | if !w.reach(t, 1500, "the source released", func() bool { |
| 402 | c := w.claim |
| 403 | return c.Op.Kind == "relocating" && c.Op.Release != nil && c.Op.Release.Box == from.ID |
| 404 | }, func() { w.worker(from) }, w.act(to), w.recoveryPass) { |
| 405 | return |
| 406 | } |
| 407 | if !w.reach(t, 1500, "serving on box 3, the source not reconciled", func() bool { |
| 408 | c := w.claim |
| 409 | return c.Op.Kind == "serving" && c.Op.Exec == to.ID |
| 410 | }, func() { w.worker(from) }, w.act(to), w.recoveryPass) { |
| 411 | return |
| 412 | } |
| 413 | w.reconcile(from) |
| 414 | w.finish(t) |
| 415 | } |
| 416 | |
| 417 | // TestDelayedSourceHoldsItsRow is the witness's schedule with its |
| 418 | // assertions: the source's row is still ready when the target completes |
| 419 | // (the release no longer holds it, ruling 28), held for the completed op |
| 420 | // after the source's next reconcile, and gone once its park-local ran. |
| 421 | func TestDelayedSourceHoldsItsRow(t *testing.T) { |
| 422 | w := newWitness() |
| 423 | w.twoBox() |
| 424 | from, to := w.boxes[0], w.boxes[3] |
| 425 | w.serve(from, 1, app0) |
| 426 | w.running(t, from) |
| 427 | w.drainTo(t, to) |
| 428 | w.reach(t, 1500, "the source released", func() bool { |
| 429 | c := w.claim |
| 430 | return c.Op.Kind == "relocating" && c.Op.Release != nil && c.Op.Release.Box == from.ID |
| 431 | }, func() { w.worker(from) }, w.act(to), w.recoveryPass) |
| 432 | w.reach(t, 1500, "serving on box 3", func() bool { return w.claim.Op.Kind == "serving" && w.claim.Op.Exec == to.ID }, func() { w.worker(from) }, w.act(to), w.recoveryPass) |
| 433 | w.mustHold(t) |
| 434 | if from.Row == nil || from.Row.Status != "ready" { |
| 435 | t.Fatalf("the delayed source's row is not the ready row its release left: %+v", from.Row) |
| 436 | } |
| 437 | op := w.claim.Op.Completed.Op |
| 438 | w.reconcile(from) |
| 439 | w.mustHold(t) |
| 440 | if from.Row == nil || from.Row.Status != "relocating" || from.Row.HoldOp != op { |
| 441 | t.Fatalf("the catch-up hold did not hold the source's row for %s: %+v", op, from.Row) |
| 442 | } |
| 443 | w.finish(t) |
| 444 | w.mustHold(t) |
| 445 | if from.Row != nil { |
| 446 | t.Fatalf("the source's row outlived its park-local: %+v", from.Row) |
| 447 | } |
| 448 | } |
| 449 | |
| 450 | // TestCatchUpHoldResidualAfterASuccessor is the residual scenario ruling |
| 451 | // 85 names and leaves open, modelled as the built code has it and not |
| 452 | // hidden: the source's reconcile is delayed, the target completes and is |
| 453 | // discharged and marked, a successor (a drain from the target, or a move |
| 454 | // to region 1) replaces the serving op, which needs only the target's |
| 455 | // discharge, and only then does the source reconcile. The successor took |
| 456 | // the completion record with it, so the catch-up hold no longer applies, |
| 457 | // and the source's row stays ready from then until its park-local |
| 458 | // retires it. No invariant fails: the gap is availability, not a second |
| 459 | // writer (the source's wake is refused by the exec check), so this is a |
| 460 | // scenario test that asserts the gap is there, not a mutation witness. |
| 461 | func TestCatchUpHoldResidualAfterASuccessor(t *testing.T) { |
| 462 | for _, successor := range []string{"relocation", "move"} { |
| 463 | t.Run(successor, func(t *testing.T) { |
| 464 | w := newWitness() |
| 465 | w.twoBox() |
| 466 | from, to := w.boxes[0], w.boxes[3] |
| 467 | w.serve(from, 1, app0) |
| 468 | w.running(t, from) |
| 469 | w.drainTo(t, to) |
| 470 | w.reach(t, 1500, "the source released", func() bool { |
| 471 | c := w.claim |
| 472 | return c.Op.Kind == "relocating" && c.Op.Release != nil && c.Op.Release.Box == from.ID |
| 473 | }, func() { w.worker(from) }, w.act(to), w.recoveryPass) |
| 474 | w.reach(t, 1500, "the completion marked, the source not reconciled", w.marked, func() { w.worker(from) }, w.act(to), w.recoveryPass, func() { w.wake(to) }) |
| 475 | w.running(t, to) |
| 476 | // the region's chores the completion queued run on box 3; only |
| 477 | // box 0's park-local stays owed (box 0's agent is the slow one) |
| 478 | w.reach(t, 600, "only the source's park-local owed", func() bool { |
| 479 | for _, o := range w.claim.Obl { |
| 480 | if !(o.Kind == "park-local" && o.Box == from.ID) { |
| 481 | return false |
| 482 | } |
| 483 | } |
| 484 | return true |
| 485 | }, w.act(to), w.recoveryPass, func() { w.wake(to) }) |
| 486 | w.mustHold(t) |
| 487 | if from.Row == nil || from.Row.Status != "ready" { |
| 488 | t.Fatalf("the delayed source's row is not the ready row its release left: %+v", from.Row) |
| 489 | } |
| 490 | want := "relocating" |
| 491 | if successor == "relocation" { |
| 492 | w.extraBox(0, 30) |
| 493 | w.userRelocate() |
| 494 | } else { |
| 495 | w.moveTo(1) |
| 496 | want = "moving" |
| 497 | } |
| 498 | if w.claim.Op.Kind != want { |
| 499 | t.Fatalf("the successor %s did not start: %s", successor, w.state()) |
| 500 | } |
| 501 | w.reconcile(from) |
| 502 | w.wake(from) |
| 503 | w.mustHold(t) |
| 504 | if from.Row == nil || from.Row.Status != "ready" || hasMain(from) { |
| 505 | t.Fatalf("the residual is not as ruling 85 describes it (row %+v, running %v)", from.Row, hasMain(from)) |
| 506 | } |
| 507 | t.Logf("residual: box %d's row is ready under %s %s, with its park-local still owed", from.ID, w.claim.Op.Kind, w.claim.Op.ID) |
| 508 | w.finish(t) |
| 509 | w.mustHold(t) |
| 510 | if from.Row != nil { |
| 511 | t.Fatalf("the source's row outlived its park-local: %+v", from.Row) |
| 512 | } |
| 513 | }) |
| 514 | } |
| 515 | } |
| 516 | |
| 517 | // TestFailedTargetCannotReleaseWithoutC1 (reloc-c1-failed-release-refused, |
| 518 | // finding 4): with claim ask C1 open, a drain's target that fails its |
| 519 | // probe cannot release in probe-failed: nothing is journaled, the |
| 520 | // operation is stuck at the release deadline, and the operator's resolve |
| 521 | // stops the target and replays, so the host serves again. The mutation |
| 522 | // releases as if C1 had landed, and this test fails. |
| 523 | func TestFailedTargetCannotReleaseWithoutC1(t *testing.T) { |
| 524 | w := twoBoxWitness() |
| 525 | from, to := w.boxes[0], w.boxes[3] |
| 526 | to.FailApp = true |
| 527 | w.serve(from, 1, app0) |
| 528 | w.running(t, from) |
| 529 | w.drainTo(t, to) |
| 530 | w.extraBox(0, 30) |
| 531 | w.reach(t, 1500, "probe-failed", func() bool { return w.claim.Op.Phase == "probe-failed" }, w.act(from, to), w.recoveryPass) |
| 532 | w.reach(t, 3*releaseDeadline, "stuck in probe-failed", func() bool { return w.claim.Op.Stuck != "" }, w.act(from, to), w.recoveryPass) |
| 533 | w.mustHold(t) |
| 534 | if c := w.claim; c.Op.Phase != "probe-failed" || c.Op.Release != nil || c.Op.Exec != to.ID || w.stats["release journaled: failed target"] != 0 { |
| 535 | t.Fatalf("the failed target released without claim ask C1: %s", w.state()) |
| 536 | } |
| 537 | if w.stats["release refused: no failed-target obligation (C1 open)"] == 0 { |
| 538 | t.Fatalf("the release was never refused: %v", w.stats) |
| 539 | } |
| 540 | w.servingOn(t, nil) |
| 541 | if w.stats["operator stopped a target that cannot release"] == 0 { |
| 542 | t.Fatalf("the operator did not stop the target: %v", w.stats) |
| 543 | } |
| 544 | w.finish(t) |
| 545 | w.mustHold(t) |
| 546 | } |
| 547 | |
| 548 | // TestLaterAttemptRefusedWithoutC11 (reloc-c11-later-attempt-refused, |
| 549 | // finding 4): with claim ask C11 open, the target's first attempt opens P |
| 550 | // and its agent crashes before the probe passes; the next attempt is |
| 551 | // refused before it is counted, and the operation is probe-failed with |
| 552 | // the count unchanged and no second admission. The mutation counts and |
| 553 | // makes it, and this test fails. |
| 554 | func TestLaterAttemptRefusedWithoutC11(t *testing.T) { |
| 555 | w := twoBoxWitness() |
| 556 | from, to := w.boxes[0], w.boxes[3] |
| 557 | w.serve(from, 1, app0) |
| 558 | w.running(t, from) |
| 559 | w.drainTo(t, to) |
| 560 | if !w.probeOpened(t, from, to) { |
| 561 | w.mustHold(t) |
| 562 | } |
| 563 | attempts, gen := w.claim.Op.Attempts, w.claim.Holder.Gen |
| 564 | w.crash(to) |
| 565 | w.reach(t, 200, "probe-failed or a second attempt", func() bool { |
| 566 | c := w.claim |
| 567 | return c.Op.Phase == "probe-failed" || c.Op.Attempts != attempts || c.Holder.Gen != gen |
| 568 | }, w.act(from, to), w.recoveryPass) |
| 569 | w.mustHold(t) |
| 570 | if c := w.claim; c.Op.Phase != "probe-failed" || c.Op.Attempts != attempts || c.Holder.Gen != gen { |
| 571 | t.Fatalf("a later attempt was counted or made without claim ask C11 (attempts %d -> %d): %s", attempts, c.Op.Attempts, w.state()) |
| 572 | } |
| 573 | if w.stats["probe-failed: a later attempt root cannot launch (C11 open)"] != 1 { |
| 574 | t.Fatalf("the refusal was not recorded once: %v", w.stats) |
| 575 | } |
| 576 | w.servingOn(t, nil) |
| 577 | w.finish(t) |
| 578 | w.mustHold(t) |
| 579 | } |
| 580 | |
| 581 | // TestCompletionBarrierExemptions (reloc-completion-barrier, ruling 77's |
| 582 | // exemptions): while box 3's completion is unmarked, a drain is refused; |
| 583 | // box 3 is then terminated, and the failover that cites its fence starts |
| 584 | // over the unmarked record and serves elsewhere. |
| 585 | func TestCompletionBarrierExemptions(t *testing.T) { |
| 586 | w := newWitness() |
| 587 | _, to, ok := w.completedDrain(t, func(to *box) { to.dischargeFails = true }) |
| 588 | if !ok { |
| 589 | w.mustHold(t) |
| 590 | } |
| 591 | w.userRelocate() |
| 592 | if w.claim.Op.Kind != "serving" || w.stats["start refused: a completed relocation is not discharged"] == 0 { |
| 593 | t.Fatalf("the drain was not refused by the barrier: %s", w.state()) |
| 594 | } |
| 595 | w.terminate(to) |
| 596 | w.reach(t, 200, "the failover started over the unmarked record", func() bool { return w.claim.Op.Kind == "relocating" }, w.actAll(), w.recoveryPass) |
| 597 | w.servingOn(t, nil) |
| 598 | w.finish(t) |
| 599 | w.mustHold(t) |
| 600 | } |
| 601 | |
| 602 | // TestProbeStopRefusesAnotherFence (reloc-probe-fence-owner as ruling 87 |
| 603 | // amends ruling 75): with another owner's fence on the host, the probe |
| 604 | // stop stops the writers and flushes and records the positions, then |
| 605 | // refuses: root is not asked (no receipt), no stop is journaled, and the |
| 606 | // other owner's fence is left as it is. |
| 607 | func TestProbeStopRefusesAnotherFence(t *testing.T) { |
| 608 | w := twoBoxWitness() |
| 609 | from, to := w.boxes[0], w.boxes[3] |
| 610 | w.serve(from, 1, app0) |
| 611 | w.running(t, from) |
| 612 | w.drainTo(t, to) |
| 613 | w.reach(t, 1500, "the probe passed", func() bool { c := w.claim; return c.Op.Phase == "probed" && c.Op.ProbeStop == nil && hasMain(to) }, w.act(from, to), w.recoveryPass) |
| 614 | to.actFence = "op-elsewhere" |
| 615 | w.drive(20, never, w.act(from, to)) |
| 616 | w.mustHold(t) |
| 617 | c := w.claim |
| 618 | if c.Op.ProbeStop != nil || to.actFence != "op-elsewhere" || to.StopEvidence[probeKey(c.Op.ID)] { |
| 619 | t.Fatalf("the refused probe stop asked root or touched the fence (stop %+v, fence %q): %s", c.Op.ProbeStop, to.actFence, w.state()) |
| 620 | } |
| 621 | if hasMain(to) || to.probeRecord == nil || to.probeRecord.Op != c.Op.ID { |
| 622 | t.Fatalf("the probe's writers were not stopped and flushed before the refusal (running %v, record %+v)", hasMain(to), to.probeRecord) |
| 623 | } |
| 624 | } |
| 625 | |
| 626 | // purgeOverATerminatedCompletionBox is seed 2224's schedule (3,000, two |
| 627 | // boxes): the completion's discharge fails, a purge closes the app's |
| 628 | // admission, and the completion box is terminated before its discharge. |
| 629 | // The failover that would cite its fence is refused by the closed |
| 630 | // admission (Admission.Register), so only the purge can move. |
| 631 | func (w *world) purgeOverATerminatedCompletionBox(t *testing.T) (to *box, ok bool) { |
| 632 | _, to, ok = w.completedDrain(t, func(to *box) { to.dischargeFails = true }) |
| 633 | if !ok { |
| 634 | return to, false |
| 635 | } |
| 636 | w.userPurge() |
| 637 | if !w.reach(t, 100, "the purge closed the app's admission", func() bool { return w.admit[app0].Closed }, w.runConsole) { |
| 638 | return to, false |
| 639 | } |
| 640 | w.terminate(to) |
| 641 | return to, true |
| 642 | } |
| 643 | |
| 644 | // wPurgeOverAFencedCompletionBox (reloc-completion-fenced-exempt, ruling |
| 645 | // 86): once a worker has written the dead completion box's fence record, |
| 646 | // the purge verifies it and starts over the unmarked completion, and the |
| 647 | // host is freed with no operator. Without the exemption the purge is |
| 648 | // refused by the barrier and the failover by the purge's closed |
| 649 | // admission, for ever (INV-3). |
| 650 | func wPurgeOverAFencedCompletionBox(t *testing.T, w *world) { |
| 651 | if _, ok := w.purgeOverATerminatedCompletionBox(t); !ok { |
| 652 | return |
| 653 | } |
| 654 | w.reach(t, 2000, "the purge freed the host", func() bool { c := w.claim; return c == nil || c.Holder == nil }, w.actAll(), w.runConsole, w.recoveryPass) |
| 655 | w.finish(t) |
| 656 | } |
| 657 | |
| 658 | // TestTerminatedCompletionBoxBlocksAPurge was the gap the model found at |
| 659 | // seed 2224, closed by ruling 86 (1b-next ca282378): the purge refused by |
| 660 | // the barrier while the completion box's fence is not yet recorded, then |
| 661 | // started over it once the fence verifies, and the host freed with no |
| 662 | // operator's declare-lost (the model's operator has none since). |
| 663 | func TestTerminatedCompletionBoxBlocksAPurge(t *testing.T) { |
| 664 | w := newWitness() |
| 665 | to, ok := w.purgeOverATerminatedCompletionBox(t) |
| 666 | if !ok { |
| 667 | w.mustHold(t) |
| 668 | } |
| 669 | w.drive(60, never, w.runConsole) // the purge's start, before any worker observed the termination |
| 670 | if c := w.claim; c.Op.Kind != "serving" || to.FenceObservedAt != 0 || w.stats["start refused: a completed relocation is not discharged"] == 0 { |
| 671 | t.Fatalf("the purge was not refused before the fence record: fence observed at %d, %s", to.FenceObservedAt, w.state()) |
| 672 | } |
| 673 | w.reach(t, 2000, "the purge freed the host", func() bool { c := w.claim; return c == nil || c.Holder == nil }, w.actAll(), w.runConsole, w.recoveryPass) |
| 674 | w.mustHold(t) |
| 675 | if to.FenceObservedAt == 0 || w.stats["start: the completion box's termination fence verified"] == 0 || w.stats["failover start: app admission closed"] == 0 { |
| 676 | t.Fatalf("not ruling 86's path: fence observed at %d, %v", to.FenceObservedAt, w.stats) |
| 677 | } |
| 678 | if contains(w.claim.Lost, to.ID) && w.claim.Holder != nil { |
| 679 | t.Fatalf("the box was declared lost: %s", w.state()) |
| 680 | } |
| 681 | } |
| 682 | |
| 683 | // TestDirtyReturnParksPreMoveFiles (moves a5e8c13f, L18 finding 1; D6): |
| 684 | // a move to region 1 rolls back dirty after its target wrote. While the |
| 685 | // target releases, the source region keeps the files live set; from the |
| 686 | // source's return CAS the newest admission (the target's) decides, so the |
| 687 | // return's install park retires the pre-move generation's manifest in the |
| 688 | // source region rather than keeping it for the at-rest sweep. INV-30 does |
| 689 | // not count it, the return serves on the source, and every invariant |
| 690 | // holds to quiescence. |
| 691 | func TestDirtyReturnParksPreMoveFiles(t *testing.T) { |
| 692 | w := newWitness() |
| 693 | w.twoBox() |
| 694 | src, sib := w.boxes[0], w.boxes[3] |
| 695 | w.serve(src, 1, app0) |
| 696 | w.running(t, src) |
| 697 | w.published(t, src) |
| 698 | loc := w.claim.Holder.Location |
| 699 | mk, _, _, _, m := w.newestManifest(src.Region, loc, w.claim.Holder.Gen) |
| 700 | if m == nil { |
| 701 | t.Fatalf("no pre-move manifest: %s", w.state()) |
| 702 | } |
| 703 | w.moveTo(1) |
| 704 | tgt := w.boxByID(w.claim.Op.Tgt) |
| 705 | w.reach(t, 600, "the target's probe runs, before the commit", func() bool { |
| 706 | c := w.claim |
| 707 | return c.Op.Kind == "landing" && !c.Op.Committed && hasMain(tgt) && w.rec.MoveState == "landing" |
| 708 | }, func() { w.worker(src); w.reconcile(src); w.worker(tgt); w.reconcile(tgt); w.runConsole() }) |
| 709 | w.mustHold(t) |
| 710 | w.appWrite(tgt) // the rollback is dirty |
| 711 | w.drive(w.claim.Op.CutDeadline+1-w.now, never, w.act(sib)) |
| 712 | w.reach(t, 1500, "the dirty return's install park done", func() bool { |
| 713 | c := w.claim |
| 714 | return c.Op.Kind == "restoring" && c.Op.Dirty && c.Op.SrcPhase != "waiting" && c.Op.SrcPhase != "claimed" && c.Op.SrcPhase != "install-park" || c.Op.Kind == "serving" |
| 715 | }, w.actAll(), w.recoveryPass, w.runConsole) |
| 716 | w.mustHold(t) |
| 717 | if w.stats["rollback-dirty"] == 0 && w.claim.Op.Kind != "restoring" { |
| 718 | t.Fatalf("the rollback was not dirty: %s", w.state()) |
| 719 | } |
| 720 | if v := w.buckets[src.Region].current(mk); v != nil { |
| 721 | t.Fatalf("the dirty return's install park kept the pre-move manifest %s (sweep owed %v): %s", mk, v.SweepOwed, w.state()) |
| 722 | } |
| 723 | w.reach(t, 1500, "the dirty return serving on the source", func() bool { c := w.claim; return c.Op.Kind == "serving" && c.Op.Exec == src.ID }, w.actAll(), w.recoveryPass, w.runConsole, func() { w.wake(src) }) |
| 724 | w.finish(t) |
| 725 | w.mustHold(t) |
| 726 | } |
| 727 | |
| 728 | // wDirtyReturnWithoutAdmission (dirty-return-needs-admission, Astra joint |
| 729 | // round 4 item 2): the source's last write is not yet replicated at the |
| 730 | // cut, so the seal does not cover its replica (src_covered false), and the |
| 731 | // move's target stalls before its probe is admitted, so the landing rolls |
| 732 | // back. The return is dirty (the seal is returned) although the target |
| 733 | // never admitted: holder.generation is still the pre-move one, the source |
| 734 | // keeps the files home, INV-30 counts the source region, and the return's |
| 735 | // install parks nothing (movesinstall.go's targetAdmitted). Without the |
| 736 | // admission check the install park retires the manifest the source's next |
| 737 | // install would pin (INV-30). |
| 738 | func wDirtyReturnWithoutAdmission(t *testing.T, w *world) { |
| 739 | w.twoBox() |
| 740 | src, sib := w.boxes[0], w.boxes[3] |
| 741 | w.serve(src, 1, app0) |
| 742 | w.running(t, src) |
| 743 | w.published(t, src) |
| 744 | w.appWrite(src) // not replicated before the cut: the seal is all that holds it |
| 745 | loc := w.claim.Holder.Location |
| 746 | w.holdKey(loc) // the source's uploads stay in flight through the seal |
| 747 | w.moveTo(1) |
| 748 | if w.claim.Op.Kind != "moving" { |
| 749 | t.Fatalf("witness: the move did not start: %s", w.state()) |
| 750 | } |
| 751 | tgt := w.boxByID(w.claim.Op.Tgt) |
| 752 | if !w.reach(t, 600, "the flip, before the target's probe", func() bool { return w.claim.Op.Kind == "landing" }, func() { w.worker(src); w.reconcile(src); w.worker(tgt); w.reconcile(tgt); w.runConsole() }) { |
| 753 | return |
| 754 | } |
| 755 | w.hold = nil |
| 756 | w.completeHeld(loc) |
| 757 | if targetAdmitted(w.claim) || w.claim.Op.SrcCovered { |
| 758 | t.Fatalf("witness: not the scenario (target admitted %v, src_covered %v): %s", targetAdmitted(w.claim), w.claim.Op.SrcCovered, w.state()) |
| 759 | } |
| 760 | // the target stalls: the landing's deadline passes with no probe |
| 761 | w.drive(w.claim.Op.CutDeadline+12-w.now, never, w.act(sib), w.runConsole, w.recoveryPass, w.operator) |
| 762 | if !w.reach(t, 1500, "the dirty return's install past its park", func() bool { |
| 763 | c := w.claim |
| 764 | return c.Op.Kind == "restoring" && c.Op.Dirty && c.Op.SrcPhase != "waiting" && c.Op.SrcPhase != "claimed" && c.Op.SrcPhase != "install-park" || c.Op.Kind == "serving" |
| 765 | }, w.actAll(), w.recoveryPass, w.runConsole) { |
| 766 | return |
| 767 | } |
| 768 | w.reach(t, 1500, "the return serving on the source", func() bool { c := w.claim; return c.Op.Kind == "serving" && c.Op.Exec == src.ID }, w.actAll(), w.recoveryPass, w.runConsole, func() { w.wake(src) }) |
| 769 | w.finish(t) |
| 770 | } |
| 771 | |
amadan