| 1 | package hostlife |
| 2 | |
| 3 | import "testing" |
| 4 | |
| 5 | // The cases region HA spec 1 §9 L18 and claim rev 6.30 §11 require of the |
| 6 | // joint model, each in a one-box and in a two-box region (bothShapes): |
| 7 | // region 0 holds box 0 alone, kept at one box by its group, or boxes 0 |
| 8 | // and 3. Each is a direct test: it drives the case through the model's |
| 9 | // own actors and asserts the outcome the spec names; the final oracle |
| 10 | // (finish) checks every invariant. The witnesses (regionha_witness_test.go) |
| 11 | // are the mutation evidence; these are the coverage. |
| 12 | |
| 13 | // serveAndPublish serves inc 1 of app A on box 0, runs it, and lets its |
| 14 | // first files manifest land, so a later failover has the host's files |
| 15 | // in S3 (finding F2 is the case where none landed). |
| 16 | func (w *world) serveAndPublish(t *testing.T) *box { |
| 17 | from := w.boxes[0] |
| 18 | w.serve(from, 1, app0) |
| 19 | w.running(t, from) |
| 20 | w.published(t, from) |
| 21 | return from |
| 22 | } |
| 23 | |
| 24 | // failedOver drives the failover of the terminated executor dead until |
| 25 | // the host serves again elsewhere, and returns the box it serves on: the |
| 26 | // peer in a two-box region, the box the group launched in a one-box |
| 27 | // region. |
| 28 | func (w *world) failedOver(t *testing.T, dead *box) *box { |
| 29 | if !w.reach(t, 3000, "serving after the failover", func() bool { |
| 30 | c := w.claim |
| 31 | return c.Op.Kind == "serving" && c.Op.Exec >= 0 && c.Op.Exec != dead.ID |
| 32 | }, w.actAll(), w.recoveryPass, w.operator, w.wakeExec) { |
| 33 | return nil |
| 34 | } |
| 35 | return w.execBox(w.claim) |
| 36 | } |
| 37 | |
| 38 | // relocTarget drives until the relocation names a target with its pin, |
| 39 | // and returns it (a one-box region's failover names the replacement the |
| 40 | // group launched once it enrolls). |
| 41 | func (w *world) relocTarget(t *testing.T) *box { |
| 42 | if !w.reach(t, 400, "a target named and pinned", func() bool { |
| 43 | c := w.claim |
| 44 | return c.Op.Kind == "relocating" && c.Op.Tgt >= 0 && c.Op.Pin != nil |
| 45 | }, w.actAll(), w.recoveryPass) { |
| 46 | return nil |
| 47 | } |
| 48 | return w.boxByID(w.claim.Op.Tgt) |
| 49 | } |
| 50 | |
| 51 | func TestL18CreateSpread(t *testing.T) { |
| 52 | bothShapes(t, func(t *testing.T, w *world, boxes int) { |
| 53 | w.serve(w.boxes[0], 1, app0) |
| 54 | w.h2 = liteHost{} |
| 55 | w.console = append(w.console, &consolePlan{Kind: "create", Host: "h2", App: app0, OpID: w.newID("cr"), Inc: 9}) |
| 56 | w.reach(t, 50, "the second host of app A placed", func() bool { return w.h2.Kind == "serving" }, w.runConsole) |
| 57 | want := 0 |
| 58 | if boxes == 2 { |
| 59 | want = 3 // the box running no instance of A (H12) |
| 60 | } |
| 61 | if w.h2.Box != want { |
| 62 | t.Fatalf("the second instance of app A landed on box %d, want %d", w.h2.Box, want) |
| 63 | } |
| 64 | if boxes == 1 { |
| 65 | return |
| 66 | } |
| 67 | w.h2 = liteHost{} |
| 68 | w.console = append(w.console, &consolePlan{Kind: "create", Host: "h2", App: app1, OpID: w.newID("cr"), Inc: 10}) |
| 69 | w.reach(t, 50, "the second host of app B placed", func() bool { return w.h2.Kind == "serving" }, w.runConsole) |
| 70 | tie := 0 |
| 71 | if tieHash(3) > tieHash(0) { |
| 72 | tie = 3 |
| 73 | } |
| 74 | if w.h2.Box != tie { |
| 75 | t.Fatalf("tie-break: app B landed on box %d, want the greater tieHash (%d)", w.h2.Box, tie) |
| 76 | } |
| 77 | }) |
| 78 | } |
| 79 | |
| 80 | func TestL18RelocationByDrain(t *testing.T) { |
| 81 | bothShapes(t, func(t *testing.T, w *world, boxes int) { |
| 82 | from := w.serveAndPublish(t) |
| 83 | wr := w.appWrite(from) |
| 84 | id := w.timerOn(from) |
| 85 | w.userRelocate() |
| 86 | if boxes == 1 { |
| 87 | // no eligible target: the drain never starts and the host keeps |
| 88 | // serving where it is (spec §4) |
| 89 | if c := w.claim; c.Op.Kind != "serving" || c.Op.Exec != from.ID { |
| 90 | t.Fatalf("a drain started in a one-box region: %s", w.state()) |
| 91 | } |
| 92 | w.finish(t) |
| 93 | return |
| 94 | } |
| 95 | to := w.peer() |
| 96 | if c := w.claim; c.Op.Kind != "relocating" || c.Op.Tgt != to.ID { |
| 97 | t.Fatalf("the drain did not start: %s", w.state()) |
| 98 | } |
| 99 | w.servingOn(t, to) |
| 100 | if w.stats["admissions covered: released"] != 1 || w.stats["writes_lost declared"] > 1 || w.stats["writes declared lost"] != 0 { |
| 101 | t.Errorf("released coverage: %d released, %d writes_lost covering %d writes (want 1, at most the first serving wake's, and 0)", w.stats["admissions covered: released"], w.stats["writes_lost declared"], w.stats["writes declared lost"]) |
| 102 | } |
| 103 | w.finish(t) |
| 104 | if !contains(to.DB.Log, wr) || !contains(to.Timers, id) && !w.delivered[id] { |
| 105 | t.Errorf("the drain lost write %d or timer %d", wr, id) |
| 106 | } |
| 107 | }) |
| 108 | } |
| 109 | |
| 110 | func TestL18TerminationDuringServing(t *testing.T) { |
| 111 | bothShapes(t, func(t *testing.T, w *world, boxes int) { |
| 112 | from := w.serveAndPublish(t) |
| 113 | wr := w.appWrite(from) |
| 114 | w.replicate(from) |
| 115 | w.completeDue(true) |
| 116 | w.terminate(from) |
| 117 | to := w.failedOver(t, from) |
| 118 | if to == nil { |
| 119 | return |
| 120 | } |
| 121 | if boxes == 2 && to != w.peer() || boxes == 1 && to.ID < 100 { |
| 122 | t.Errorf("served on box %d after the failover (one-box regions wait for the group's replacement)", to.ID) |
| 123 | } |
| 124 | w.finish(t) |
| 125 | if to.DB == nil || !contains(to.DB.Log, wr) { |
| 126 | t.Errorf("the failover lost the replicated write %d", wr) |
| 127 | } |
| 128 | }) |
| 129 | } |
| 130 | |
| 131 | func TestL18NestedFailover(t *testing.T) { |
| 132 | bothShapes(t, func(t *testing.T, w *world, boxes int) { |
| 133 | from := w.serveAndPublish(t) |
| 134 | w.extraBox(0, spareID()) |
| 135 | var to *box |
| 136 | if boxes == 2 { |
| 137 | w.drainTo(t, nil) |
| 138 | to = w.boxByID(w.claim.Op.Tgt) |
| 139 | } else { |
| 140 | w.terminate(from) |
| 141 | to = w.relocTarget(t) |
| 142 | } |
| 143 | if to == nil || !w.reach(t, 1500, "the probe runs on the target", func() bool { return preparedOpened(w.claim) && hasMain(to) }, w.actAll(), w.recoveryPass, func() { w.wake(to) }) { |
| 144 | return |
| 145 | } |
| 146 | wr := w.appWrite(to) |
| 147 | w.replicate(to) |
| 148 | w.completeDue(true) // the target's write is in P's remote before it dies |
| 149 | w.terminate(to) |
| 150 | next := w.failedOver(t, to) |
| 151 | w.finish(t) |
| 152 | if next == nil || next.DB == nil || !contains(next.DB.Log, wr) { |
| 153 | t.Fatalf("the nested failover lost the target's replicated write %d", wr) |
| 154 | } |
| 155 | if w.stats["nested failover after P was admitted"] != 1 { |
| 156 | t.Fatalf("no nested failover: %v", w.stats) |
| 157 | } |
| 158 | }) |
| 159 | } |
| 160 | |
| 161 | func TestL18StaleEdgesFile(t *testing.T) { |
| 162 | bothShapes(t, func(t *testing.T, w *world, boxes int) { |
| 163 | from := w.serveAndPublish(t) |
| 164 | if boxes == 1 { |
| 165 | // a one-box region's former executor is gone: a request through |
| 166 | // its stale edges file reaches nothing |
| 167 | w.terminate(from) |
| 168 | w.failedOver(t, from) |
| 169 | w.drive(30, never, func() { w.reconcile(from); w.wake(from) }) |
| 170 | if hasMain(from) { |
| 171 | t.Fatalf("the terminated box ran the host") |
| 172 | } |
| 173 | w.finish(t) |
| 174 | return |
| 175 | } |
| 176 | to := w.peer() |
| 177 | w.drainTo(t, to) |
| 178 | w.servingOn(t, to) |
| 179 | if from.Row == nil || from.Row.Status == "ready" { |
| 180 | t.Fatalf("the former executor's row is not held: %+v", from.Row) |
| 181 | } |
| 182 | w.drive(30, never, func() { w.reconcile(from); w.wake(from) }) |
| 183 | if hasMain(from) { |
| 184 | t.Fatalf("a request through the stale edges file started the host on the former executor") |
| 185 | } |
| 186 | w.finish(t) |
| 187 | }) |
| 188 | } |
| 189 | |
| 190 | func TestL18UnenrolledReference(t *testing.T) { |
| 191 | bothShapes(t, func(t *testing.T, w *world, boxes int) { |
| 192 | from := w.serveAndPublish(t) |
| 193 | if p := w.peer(); p != nil { |
| 194 | p.HasKey = false // the region's other box may work, never take the host (R9) |
| 195 | } |
| 196 | w.terminate(from) |
| 197 | w.now += replaceDelay |
| 198 | w.launchReplacements() |
| 199 | nb := w.boxes[len(w.boxes)-1] |
| 200 | if nb.Region != 0 || nb.Enrolled { |
| 201 | t.Fatalf("no unenrolled replacement in region 0: %+v", nb) |
| 202 | } |
| 203 | nb.EnrollAt = w.now + 10*enrollDelay // its confirmation is slow |
| 204 | w.drive(3*enrollDelay, never, w.actAll(), w.recoveryPass) |
| 205 | c := w.claim |
| 206 | if c.Op.Exec == nb.ID || c.Op.Tgt == nb.ID { |
| 207 | t.Fatalf("an unenrolled box was named: %s", w.state()) |
| 208 | } |
| 209 | if boxes == 2 && (c.Op.Kind != "serving" || contains(c.Lost, from.ID)) { |
| 210 | // Worker.Start needs an eligible target (rev 7, Astra round 4 |
| 211 | // finding 3): the failover does not start, and the claim still |
| 212 | // names the dead box, until the replacement enrolls |
| 213 | t.Fatalf("the failover started with no eligible target: %s", w.state()) |
| 214 | } |
| 215 | if boxes == 1 && c.Op.Kind != "serving" { |
| 216 | // no enrolled worker of the region exists to write the fence: the |
| 217 | // claim still names the dead box until the replacement enrolls |
| 218 | t.Fatalf("a one-box region was fenced with no enrolled worker in it: %s", w.state()) |
| 219 | } |
| 220 | nb.EnrollAt = w.now - enrollDelay |
| 221 | if to := w.failedOver(t, from); to != nb { |
| 222 | t.Fatalf("served on %v, want the replacement %d once enrolled", to, nb.ID) |
| 223 | } |
| 224 | w.finish(t) |
| 225 | }) |
| 226 | } |
| 227 | |
| 228 | // TestL18AdverseVolume (claim §11's adverse-volume case; H19; rev 5, Astra |
| 229 | // round 2 finding 2): the current executor is terminated with a |
| 230 | // volume that was not DeleteOnTermination. Execution recovery proceeds |
| 231 | // to serving on the region's other box (two-box) or on the group's |
| 232 | // replacement (one-box) with no operator, and the dead box's park-local |
| 233 | // stays open throughout; only the operator's cleanup (the declared |
| 234 | // destroyed-local loss) removes it. |
| 235 | func TestL18AdverseVolume(t *testing.T) { |
| 236 | bothShapes(t, func(t *testing.T, w *world, boxes int) { |
| 237 | from := w.serveAndPublish(t) |
| 238 | from.DeleteOnTermination = false |
| 239 | w.terminate(from) // the current executor |
| 240 | if !w.reach(t, 3000, "serving after the failover, with no operator", func() bool { |
| 241 | c := w.claim |
| 242 | return c.Op.Kind == "serving" && c.Op.Exec >= 0 && c.Op.Exec != from.ID |
| 243 | }, w.actAll(), w.recoveryPass, w.wakeExec) { |
| 244 | return |
| 245 | } |
| 246 | to := w.execBox(w.claim) |
| 247 | if boxes == 2 && to != w.peer() { |
| 248 | t.Fatalf("served on %v, want the region's other box %d", to, w.peer().ID) |
| 249 | } |
| 250 | if !w.owesLocal(w.claim, from) { |
| 251 | t.Fatalf("the dead box's park-local is not open after the failover: %s", w.state()) |
| 252 | } |
| 253 | if w.stats["fence: park-local chores left open (volume evidence)"] == 0 || w.stats["fence: park-local chores discharged"] != 0 { |
| 254 | t.Fatalf("the fence's discharge ignored the volume evidence: %v", w.stats) |
| 255 | } |
| 256 | w.drive(200, never, w.actAll(), w.recoveryPass, w.wakeExec) |
| 257 | if !w.owesLocal(w.claim, from) { |
| 258 | t.Fatalf("the dead box's park-local closed without the operator: %s", w.state()) |
| 259 | } |
| 260 | w.finish(t) // the operator's destroyed-local declaration clears it |
| 261 | if w.owesLocal(w.claim, from) { |
| 262 | t.Fatalf("the park-local is still open after the operator's cleanup: %s", w.state()) |
| 263 | } |
| 264 | }) |
| 265 | } |
| 266 | |
| 267 | func TestL18CrashDuringInstallation(t *testing.T) { |
| 268 | for _, boxes := range []int{1, 2} { |
| 269 | for _, at := range []string{"installing", "installed", "probing"} { |
| 270 | t.Run(map[int]string{1: "one-box region", 2: "two-box region"}[boxes]+"/"+at, func(t *testing.T) { |
| 271 | w := regionWorld(boxes) |
| 272 | from := w.serveAndPublish(t) |
| 273 | w.timerOn(from) |
| 274 | if boxes == 2 { |
| 275 | w.drainTo(t, nil) |
| 276 | } else { |
| 277 | w.terminate(from) |
| 278 | } |
| 279 | w.relocPhase(t, at, w.actAll(), w.recoveryPass) |
| 280 | to := w.boxByID(w.claim.Op.Tgt) |
| 281 | w.drive(2, never, w.act(to)) // a step into the phase: a PUT issued, a files directory extracted |
| 282 | w.crash(to) |
| 283 | if c := w.claim; c.Op.Kind == "relocating" && c.Op.Phase == "probed" && c.Op.ProbeStop == nil && to.probeRecord == nil { |
| 284 | // the probe passed and the crash came before the probe |
| 285 | // stop recorded its flush (ruling 87): no replicator, so |
| 286 | // the stop refuses until the deadline, and the operator's |
| 287 | // resolve serves the host elsewhere |
| 288 | w.servingOn(t, nil) |
| 289 | } else { |
| 290 | w.servingOn(t, to) |
| 291 | } |
| 292 | w.finish(t) |
| 293 | w.mustHold(t) |
| 294 | }) |
| 295 | } |
| 296 | } |
| 297 | } |
| 298 | |
| 299 | func TestL18LateFileUploads(t *testing.T) { |
| 300 | bothShapes(t, func(t *testing.T, w *world, boxes int) { |
| 301 | from := w.serveAndPublish(t) |
| 302 | late := w.timerOn(from) |
| 303 | loc := w.claim.Holder.Location |
| 304 | w.holdKey(loc + "files/") |
| 305 | w.publishFiles(from) // the manifest carrying the timer, held |
| 306 | w.hold = nil |
| 307 | w.terminate(from) |
| 308 | to := w.failedOver(t, from) |
| 309 | if to == nil { |
| 310 | return |
| 311 | } |
| 312 | if contains(to.Timers, late) { |
| 313 | t.Fatalf("the install applied a manifest that had not landed at the pin") |
| 314 | } |
| 315 | declared := false |
| 316 | for _, a := range w.acks { |
| 317 | if a.W == late && a.Exempt { |
| 318 | declared = true |
| 319 | } |
| 320 | } |
| 321 | if !declared { |
| 322 | t.Fatalf("the timer the late manifest carried was not declared with the fence") |
| 323 | } |
| 324 | if n := w.completeHeld(loc + "files/"); n != 1 { |
| 325 | t.Fatalf("held manifest PUTs completed: %d, want 1", n) |
| 326 | } |
| 327 | w.finish(t) |
| 328 | }) |
| 329 | } |
| 330 | |
| 331 | func TestL18LatePutAfterTermination(t *testing.T) { |
| 332 | bothShapes(t, func(t *testing.T, w *world, boxes int) { |
| 333 | from := w.serveAndPublish(t) |
| 334 | late := w.appWrite(from) |
| 335 | loc := w.claim.Holder.Location |
| 336 | w.holdKey(loc) |
| 337 | w.replicate(from) // the upload of the late write is held |
| 338 | w.hold = nil |
| 339 | w.terminate(from) |
| 340 | to := w.failedOver(t, from) |
| 341 | if to == nil { |
| 342 | return |
| 343 | } |
| 344 | if contains(to.DB.Log, late) { |
| 345 | t.Fatalf("the install restored a write whose upload had not landed at the pin") |
| 346 | } |
| 347 | if n := w.completeHeld(loc); n == 0 { |
| 348 | t.Fatalf("no held upload to complete") |
| 349 | } |
| 350 | w.finish(t) // the late object lands in generation_from, read by nothing, parked |
| 351 | }) |
| 352 | } |
| 353 | |
| 354 | // ---------- claim §11's region HA witnesses beyond L18 (two-box) ---------- |
| 355 | |
| 356 | // TestRelocationStartRacesPurge: a worker read the serving claim, then |
| 357 | // paused; a purge started meanwhile. A fresh start reads the purging |
| 358 | // claim and refuses; no relocation runs over a purging host. |
| 359 | func TestRelocationStartRacesPurge(t *testing.T) { |
| 360 | w := twoBoxWitness() |
| 361 | w.serve(w.boxes[0], 1, app0) |
| 362 | w.userPurge() |
| 363 | w.reach(t, 100, "the purge started", func() bool { return w.claim.Op.Kind == "purging" || w.admit[app0].Closed }, w.runConsole) |
| 364 | w.userRelocate() |
| 365 | if w.claim.Op.Kind == "relocating" { |
| 366 | t.Fatalf("a relocation started over a purging host: %s", w.state()) |
| 367 | } |
| 368 | w.finish(t) |
| 369 | w.mustHold(t) |
| 370 | } |
| 371 | |
| 372 | // TestEmptyDatabaseDrain: a created host whose database never committed |
| 373 | // (nothing replicated) but has files and a timer is drained: the release |
| 374 | // flushes at 0, the pin is an evidenced empty one, and the files and the |
| 375 | // timer arrive through the manifests. |
| 376 | func TestEmptyDatabaseDrain(t *testing.T) { |
| 377 | w := twoBoxWitness() |
| 378 | w.console = append(w.console, &consolePlan{Kind: "create", Host: "h", App: app0, OpID: w.newID("cr"), Inc: 1, Region: 0}) |
| 379 | w.reach(t, 100, "created", func() bool { c := w.claim; return c != nil && c.Holder != nil && c.Op.Kind == "serving" }, w.runConsole, w.recoveryPass) |
| 380 | from := w.execBox(w.claim) |
| 381 | w.reach(t, 100, "the host runs", func() bool { return hasMain(from) }, w.actAll(), func() { w.wake(from) }) |
| 382 | id := w.timerOn(from) |
| 383 | key := from.Files.Key |
| 384 | w.published(t, from) |
| 385 | w.drainTo(t, nil) |
| 386 | to := w.boxByID(w.claim.Op.Tgt) |
| 387 | w.servingOn(t, to) |
| 388 | w.finish(t) |
| 389 | w.mustHold(t) |
| 390 | if !contains(to.Timers, id) && !w.delivered[id] { |
| 391 | t.Fatalf("the timer did not survive the empty-database drain") |
| 392 | } |
| 393 | if to.Files == nil || to.Files.Key != key { |
| 394 | t.Fatalf("the files did not survive the empty-database drain: %+v", to.Files) |
| 395 | } |
| 396 | } |
| 397 | |
| 398 | // TestReleasedThenFailedProbe: the drain's first probe admission is |
| 399 | // covered released; the target cannot run the app, fails its probe, |
| 400 | // releases, and the nested relocation's admission is covered by the |
| 401 | // failed target's own release: nothing is declared. |
| 402 | func TestReleasedThenFailedProbe(t *testing.T) { |
| 403 | w := twoBoxWitness() |
| 404 | w.claimC1 = true // the failed target's release (claim ask C1, finding 4) |
| 405 | from, to := w.boxes[0], w.boxes[3] |
| 406 | to.FailApp = true |
| 407 | w.serve(from, 1, app0) |
| 408 | w.running(t, from) |
| 409 | w.drainTo(t, to) |
| 410 | third := w.extraBox(0, 30) |
| 411 | w.servingOn(t, third) |
| 412 | w.finish(t) |
| 413 | w.mustHold(t) // nothing acknowledged is lost: each later attempt declares its predecessor's session, which acknowledged nothing |
| 414 | if w.stats["admissions covered: released"] < 1 { |
| 415 | t.Fatalf("the drain's first probe admission was not covered released: %v", w.stats) |
| 416 | } |
| 417 | } |
| 418 | |
| 419 | // TestStuckAfterTwoReplacements (reloc-replacement-cap, an operator |
| 420 | // bound): three targets die at installing; the third death is the |
| 421 | // exhausted fence, the operation is stuck, and the operator's resolve |
| 422 | // replays it onto a spare. |
| 423 | func TestStuckAfterTwoReplacements(t *testing.T) { |
| 424 | w := twoBoxWitness() |
| 425 | from := w.boxes[0] |
| 426 | w.serve(from, 1, app0) |
| 427 | w.running(t, from) |
| 428 | w.extraBox(0, 30) |
| 429 | w.extraBox(0, 31) |
| 430 | w.drainTo(t, nil) |
| 431 | for i := 0; i < 3; i++ { |
| 432 | if !w.reach(t, 1500, "a target installing", func() bool { |
| 433 | c := w.claim |
| 434 | return c.Op.Kind == "relocating" && c.Op.Phase == "installing" && c.Op.Stuck == "" |
| 435 | }, w.act(w.liveBoxes()...), w.recoveryPass) { |
| 436 | t.Fatalf("target %d never installed: %s", i, w.state()) |
| 437 | } |
| 438 | x := w.boxByID(w.claim.Op.Exec) |
| 439 | w.terminate(x) |
| 440 | w.reach(t, 100, "fenced", func() bool { return contains(w.claim.Lost, x.ID) }, w.act(w.liveBoxes()...)) |
| 441 | } |
| 442 | if w.claim.Op.Stuck == "" { |
| 443 | t.Fatalf("a third loss did not make the operation stuck: %s", w.state()) |
| 444 | } |
| 445 | w.extraBox(0, spareID()) |
| 446 | w.servingOn(t, nil) // the operator resolves it |
| 447 | w.finish(t) |
| 448 | w.mustHold(t) |
| 449 | if w.stats["operator resolved a stuck relocation"] != 1 { |
| 450 | t.Fatalf("the operator did not resolve it: %v", w.stats) |
| 451 | } |
| 452 | } |
| 453 | |
| 454 | // TestReleaseStopsAnEscapedDescendant (L10): a process that escaped the |
| 455 | // cgroup is stopped with the whole tree before release step 1 passes |
| 456 | // (the model's evidence-whole-tree: the cleanup-only reopen kills the |
| 457 | // cgroup and every identified stray); the source's release is journaled |
| 458 | // only once nothing of the host runs on it. |
| 459 | func TestReleaseStopsAnEscapedDescendant(t *testing.T) { |
| 460 | w := twoBoxWitness() |
| 461 | from, to := w.boxes[0], w.boxes[3] |
| 462 | w.serve(from, 1, app0) |
| 463 | w.running(t, from) |
| 464 | from.Procs = append(from.Procs, &proc{Main: false, Strict: false, Inc: 1, Box: from.ID, Session: w.claim.Holder.Session}) |
| 465 | w.drainTo(t, to) |
| 466 | if !w.reach(t, 300, "the source's release journaled", func() bool { return w.claim.Op.Release != nil }, w.act(from, to), w.recoveryPass) { |
| 467 | w.mustHold(t) |
| 468 | } |
| 469 | if len(from.Procs) != 0 { |
| 470 | t.Fatalf("the release was journaled with %d processes of the host alive on the source", len(from.Procs)) |
| 471 | } |
| 472 | w.servingOn(t, to) |
| 473 | w.finish(t) |
| 474 | w.mustHold(t) |
| 475 | } |
| 476 | |
| 477 | // ---------- the bounds: direct evidence for the rules no invariant catches alone ---------- |
| 478 | |
| 479 | // TestDrainRefusedForANeverWokenHost (reloc-drain-needs-writer, ruling |
| 480 | // 11): a host with no writer session is never drained; its failover |
| 481 | // relocates it. |
| 482 | func TestDrainRefusedForANeverWokenHost(t *testing.T) { |
| 483 | w := twoBoxWitness() |
| 484 | w.serve(w.boxes[0], 1, app0) |
| 485 | w.cas(w.claim, func(n *claim) bool { n.Holder.Session = 0; return true }) |
| 486 | w.userRelocate() |
| 487 | if w.claim.Op.Kind != "serving" || w.stats["drain refused: the host never had a writer"] != 1 { |
| 488 | t.Fatalf("a never-woken host was drained: %s", w.state()) |
| 489 | } |
| 490 | } |
| 491 | |
| 492 | // TestReplacedTargetAcknowledgesBeforePinning (reloc-ack-before-pinning): |
| 493 | // the target is replaced in releasing; the operation leaves releasing |
| 494 | // only once the replacement's execution lease is current. |
| 495 | func TestReplacedTargetAcknowledgesBeforePinning(t *testing.T) { |
| 496 | w := twoBoxWitness() |
| 497 | from, to := w.boxes[0], w.boxes[3] |
| 498 | w.serve(from, 1, app0) |
| 499 | w.running(t, from) |
| 500 | w.published(t, from) |
| 501 | w.drainTo(t, to) |
| 502 | spare := w.extraBox(0, 30) |
| 503 | w.relocPhase(t, "releasing", w.act(from, to), w.recoveryPass) |
| 504 | w.terminate(to) |
| 505 | w.reach(t, 100, "the target replaced in releasing", func() bool { return w.claim.Op.Tgt == spare.ID }, w.act(from)) |
| 506 | // the spare never acts: no lease, so the operation must stay in releasing |
| 507 | w.drive(40, never, w.act(from)) |
| 508 | if c := w.claim; c.Op.Phase != "releasing" || w.stats["pin after release: waits for the acknowledgements"] == 0 { |
| 509 | t.Fatalf("left releasing without the replacement's acknowledgement: %s", w.state()) |
| 510 | } |
| 511 | w.servingOn(t, spare) |
| 512 | w.finish(t) |
| 513 | w.mustHold(t) |
| 514 | } |
| 515 | |
| 516 | // TestProbeAttemptsCountedBeforeTheyAreMade (reloc-probe-attempts, |
| 517 | // ruling 19): an agent that crashes in every attempt never buys a |
| 518 | // fourth; the target goes to probe-failed after three. |
| 519 | func TestProbeAttemptsCountedBeforeTheyAreMade(t *testing.T) { |
| 520 | w := twoBoxWitness() |
| 521 | from, to := w.boxes[0], w.boxes[3] |
| 522 | w.serve(from, 1, app0) |
| 523 | w.running(t, from) |
| 524 | w.drainTo(t, to) |
| 525 | w.relocPhase(t, "probing", w.act(from, to), w.recoveryPass) |
| 526 | to.envUnreadyFor = 1000 // the launch environment never gets ready: each attempt is still starting when the agent dies |
| 527 | before := w.stats["admissions: cold"] |
| 528 | for i := 0; i < 8 && w.claim.Op.Phase == "probing"; i++ { |
| 529 | w.drive(3, func() bool { return to.open != nil }, w.act(to)) |
| 530 | w.crash(to) // each attempt dies before its probe |
| 531 | } |
| 532 | if n := w.stats["admissions: cold"] - before; n > probeTries || w.claim.Op.Phase == "probing" { |
| 533 | t.Fatalf("%d attempts admitted, phase %s; want at most %d, then probe-failed", n, w.claim.Op.Phase, probeTries) |
| 534 | } |
| 535 | } |
| 536 | |
| 537 | // TestCreateNamesItsExecutorOnce (create-choice-once, claim P5-D2, |
| 538 | // P5-D6): a create names its executor in the intent; when that box is |
| 539 | // gone before the create completes, recovery cancels the create and |
| 540 | // never substitutes the region's other box. |
| 541 | func TestCreateNamesItsExecutorOnce(t *testing.T) { |
| 542 | w := twoBoxWitness() |
| 543 | p := &consolePlan{Kind: "create", Host: "h", App: app0, OpID: w.newID("cr"), Inc: 1, Region: 0} |
| 544 | w.admit[app0].Reg[p.OpID] = regEntry{Until: w.now + 1000} |
| 545 | w.console = append(w.console, p) |
| 546 | w.reach(t, 50, "the create claimed the host", func() bool { c := w.claim; return c != nil && c.Op.Kind == "creating" }, w.runConsole) |
| 547 | chosen := w.boxByID(w.claim.Op.NewBox) |
| 548 | w.terminate(chosen) |
| 549 | w.drive(200, func() bool { c := w.claim; return c == nil || c.Holder == nil || c.Op.Kind == "serving" }, w.runConsole, w.recoveryPass) |
| 550 | if c := w.claim; c != nil && c.Holder != nil && c.Op.Kind == "serving" { |
| 551 | t.Fatalf("the create substituted box %d for its chosen box %d", c.Op.Exec, chosen.ID) |
| 552 | } |
| 553 | if w.stats["create cancelled: its box is gone"] != 1 { |
| 554 | t.Fatalf("the create was not cancelled: %v", w.stats) |
| 555 | } |
| 556 | } |
| 557 | |
| 558 | // TestStoppedBoxNeverFenced (fence-terminated-only): a stopped executor |
| 559 | // is never fenced; when it starts again with its disk, it serves. |
| 560 | func TestStoppedBoxNeverFenced(t *testing.T) { |
| 561 | w := twoBoxWitness() |
| 562 | from, to := w.boxes[0], w.boxes[3] |
| 563 | w.serve(from, 1, app0) |
| 564 | w.running(t, from) |
| 565 | from.Destroyed, from.Procs = true, nil // stopped, not terminated |
| 566 | w.drive(100, never, w.act(to), w.recoveryPass) |
| 567 | if contains(w.claim.Lost, from.ID) || w.claim.Op.Kind != "serving" { |
| 568 | t.Fatalf("a stopped box was fenced: %s", w.state()) |
| 569 | } |
| 570 | from.Destroyed = false |
| 571 | w.running(t, from) |
| 572 | w.finish(t) |
| 573 | w.mustHold(t) |
| 574 | } |
| 575 | |
| 576 | // TestRelocationStartGuardRefuses (reloc-start-guard): a whole-location |
| 577 | // chore in the region refuses the start; a park-generation does not. |
| 578 | func TestRelocationStartGuardRefuses(t *testing.T) { |
| 579 | w := twoBoxWitness() |
| 580 | w.serve(w.boxes[0], 1, app0) |
| 581 | w.claim.Obl = append(w.claim.Obl, obligation{ID: "pg", Op: "old", Kind: "park-generation", Region: 0, Box: -1, Prefix: w.claim.Holder.Location, Ret: "old"}) |
| 582 | if !w.relocationStartAllowed(w.claim, w.boxes[0], w.boxes[3]) { |
| 583 | t.Fatalf("a park-generation refused the start (L11)") |
| 584 | } |
| 585 | w.claim.Obl = append(w.claim.Obl, obligation{ID: "pr", Op: "old", Kind: "park-replica", Region: 0, Box: -1, Prefix: w.claim.Holder.Location}) |
| 586 | if w.relocationStartAllowed(w.claim, w.boxes[0], w.boxes[3]) { |
| 587 | t.Fatalf("a park-replica in the region did not refuse the start (CC11)") |
| 588 | } |
| 589 | } |
| 590 | |
| 591 | // TestOnlyTheExecutorInstalls (reloc-exec-only-installs, HA2): through a |
| 592 | // drain, no box but op.exec writes P or makes a local install. |
| 593 | func TestOnlyTheExecutorInstalls(t *testing.T) { |
| 594 | w := twoBoxWitness() |
| 595 | from, to := w.boxes[0], w.boxes[3] |
| 596 | w.serve(from, 1, app0) |
| 597 | w.running(t, from) |
| 598 | w.drainTo(t, to) |
| 599 | w.servingOn(t, to) |
| 600 | if from.receipt != nil || from.Files != nil && from.Files.Gen != "" && from.Files.Gen == w.claim.Holder.Gen { |
| 601 | t.Fatalf("the source made a local install of P") |
| 602 | } |
| 603 | } |
| 604 | |
| 605 | // ---------- findings F1 to F3, closed by 1b-next rev 17 to 19 ---------- |
| 606 | |
| 607 | // TestFailoverBeforeTheFirstManifest (finding F2, closed by 1b-next |
| 608 | // ruling 25 as rev 18 amended it): a host none of whose files manifests |
| 609 | // has landed is terminated. The pin refuses files that carry no |
| 610 | // instance.key for a nonempty database, the operation is stuck, and |
| 611 | // nothing is installed: no fresh key over sealed data. The database is |
| 612 | // unrecoverable by relocation (L21); the operator's replay never accepts |
| 613 | // the keyless pin. Under the mutation of files-pin-needs-key the install |
| 614 | // mints a fresh key (INV-4) and this test fails. |
| 615 | func TestFailoverBeforeTheFirstManifest(t *testing.T) { |
| 616 | w := twoBoxWitness() |
| 617 | from := w.boxes[0] |
| 618 | w.serve(from, 1, app0) |
| 619 | loc := w.claim.Holder.Location |
| 620 | w.holdKey(loc + "files/") // the first manifest's PUT has not landed |
| 621 | w.running(t, from) |
| 622 | w.hold = nil |
| 623 | w.terminate(from) |
| 624 | w.drive(400, func() bool { return len(w.violations) > 0 }, w.actAll(), w.recoveryPass, w.wakeExec) |
| 625 | w.mustHold(t) |
| 626 | c := w.claim |
| 627 | if c.Op.Kind != "relocating" || c.Op.Stuck == "" || c.Op.Pin != nil || c.Op.Prepared != "" { |
| 628 | t.Fatalf("the failover did not stop at an unevidenced files pin: %s", w.state()) |
| 629 | } |
| 630 | if w.stats["pin: files unevidenced, no instance.key"] == 0 { |
| 631 | t.Fatalf("the pin never refused: %v", w.stats) |
| 632 | } |
| 633 | } |
| 634 | |
| 635 | // TestProbeFailedInATwoBoxRegion (finding F1, closed by 1b-next ruling |
| 636 | // 26): a drain's target cannot run the app; its probe fails and it |
| 637 | // releases. With no spare the worker waits, and ReleaseDeadline after |
| 638 | // probe_failed_since the operation is stuck; nothing names the source |
| 639 | // automatically. The operator's resolve returns the host to the source |
| 640 | // (plan ruling 13). Under the mutation of reloc-no-spare-bound the |
| 641 | // operation waits for ever (INV-3) and this test fails. |
| 642 | func TestProbeFailedInATwoBoxRegion(t *testing.T) { |
| 643 | w := twoBoxWitness() |
| 644 | w.claimC1 = true // the failed target's release (claim ask C1, finding 4) |
| 645 | from, to := w.boxes[0], w.boxes[3] |
| 646 | to.FailApp = true |
| 647 | w.serve(from, 1, app0) |
| 648 | w.running(t, from) |
| 649 | w.drainTo(t, to) |
| 650 | w.reach(t, 1500, "probe-failed with the target released", func() bool { |
| 651 | c := w.claim |
| 652 | return c.Op.Phase == "probe-failed" && c.Op.Release != nil && c.Op.Release.Box == to.ID |
| 653 | }, w.actAll(), w.recoveryPass) |
| 654 | w.drive(releaseDeadline/2, never, w.actAll(), w.recoveryPass) |
| 655 | if c := w.claim; c.Op.Stuck != "" || c.Op.Tgt == from.ID { |
| 656 | t.Fatalf("before the deadline the operation is stuck or names the source: %s", w.state()) |
| 657 | } |
| 658 | w.reach(t, 1500, "stuck with no spare", func() bool { return w.claim.Op.Stuck == "no eligible box for the nested relocation" }, w.actAll(), w.recoveryPass) |
| 659 | w.servingOn(t, from) |
| 660 | w.finish(t) |
| 661 | w.mustHold(t) |
| 662 | } |
| 663 | |
| 664 | // TestAbandonedPreparedGenerationParked (finding F3, closed by 1b-next |
| 665 | // ruling 27 and rev 18 finding 3): an unopened P is recorded on every |
| 666 | // path that drops it and parked at completion. installing: the target |
| 667 | // is replaced at installing and the replacement builds P'. probe-failed: |
| 668 | // the probe deadline passes before any admission, the target is then |
| 669 | // terminated, and the replacement table's nested branch drops P. Under |
| 670 | // the mutation of reloc-abandoned-prepared INV-10 finds P's objects. |
| 671 | func TestAbandonedPreparedGenerationParked(t *testing.T) { |
| 672 | for _, path := range []string{"installing", "probe-failed"} { |
| 673 | t.Run(path, func(t *testing.T) { |
| 674 | w := twoBoxWitness() |
| 675 | from, to := w.boxes[0], w.boxes[3] |
| 676 | w.serve(from, 1, app0) |
| 677 | w.running(t, from) |
| 678 | w.published(t, from) |
| 679 | w.drainTo(t, to) |
| 680 | third := w.extraBox(0, 30) |
| 681 | w.reach(t, 1500, "P's snapshot written", func() bool { c := w.claim; return c.Op.Phase == "installing" && c.Op.BaseKey != "" }, w.act(from, to), w.recoveryPass) |
| 682 | p := w.claim.Op.Prepared |
| 683 | if path == "probe-failed" { |
| 684 | w.dropPBeforeAdmission(t, from, to) |
| 685 | } else { |
| 686 | w.terminate(to) |
| 687 | } |
| 688 | w.reach(t, 300, "P dropped", func() bool { return w.claim.Op.Prepared != p }, w.actAll(), w.recoveryPass) |
| 689 | w.servingOn(t, third) |
| 690 | if w.stats["relocation: abandoned prepared generation parked"] == 0 { |
| 691 | t.Fatalf("the abandoned P %q was not parked at serving", p) |
| 692 | } |
| 693 | w.finish(t) |
| 694 | w.mustHold(t) |
| 695 | }) |
| 696 | } |
| 697 | } |
| 698 | |
| 699 | // dropPBeforeAdmission is rev 18 finding 3's trace: the target freezes |
| 700 | // in probing until the probe deadline passes with P unopened, the worker |
| 701 | // moves it to probe-failed, and it is terminated there. |
| 702 | func (w *world) dropPBeforeAdmission(t *testing.T, from, to *box) { |
| 703 | w.reach(t, 1500, "probing", func() bool { return w.claim.Op.Phase == "probing" }, w.act(from, to), w.recoveryPass) |
| 704 | to.Frozen = true |
| 705 | w.reach(t, 3*probeDeadline, "probe-failed before any admission", func() bool { return w.claim.Op.Phase == "probe-failed" }, w.act(from), w.recoveryPass) |
| 706 | if preparedOpened(w.claim) { |
| 707 | t.Fatalf("P was opened") |
| 708 | } |
| 709 | w.terminate(to) |
| 710 | } |
| 711 | |
| 712 | // invNumber is the n of the first "INV-n" in a violation, 0 if none. |
| 713 | func invNumber(v string) int { |
| 714 | for i := 0; i+4 < len(v); i++ { |
| 715 | if v[i:i+4] == "INV-" { |
| 716 | n := 0 |
| 717 | for j := i + 4; j < len(v) && v[j] >= '0' && v[j] <= '9'; j++ { |
| 718 | n = n*10 + int(v[j]-'0') |
| 719 | } |
| 720 | return n |
| 721 | } |
| 722 | } |
| 723 | return 0 |
| 724 | } |
| 725 | |
| 726 | // mustHold fails the test, with the trace, if any invariant was violated. |
| 727 | func (w *world) mustHold(t *testing.T) { |
| 728 | t.Helper() |
| 729 | if len(w.violations) > 0 { |
| 730 | dump(t, w) |
| 731 | t.Fatalf("%s", w.violations[0]) |
| 732 | } |
| 733 | } |
| 734 | |
amadan