| index 3f56952b1..6cc45814f 100644 |
| --- a/internal/sim/hostlife/regionha_round1_test.go |
| +++ b/internal/sim/hostlife/regionha_round1_test.go |
| @@ -73,9 +73,23 @@ func (w *world) stepUntil(t *testing.T, max int, what string, done func() bool) |
| return false |
| } |
| |
| -// atPhase: the relocation is exactly in phase p (not beyond it). |
| +// atPhase: the relocation is exactly in phase p (not beyond it), as |
| +// 1b-next names it: an initial failover at pinning with PinFailover set |
| +// is in starting (rev 7, Astra round 4 finding 2). |
| func (w *world) atPhase(p string) func() bool { |
| - return func() bool { c := w.claim; return c.Op.Kind == "relocating" && c.Op.Phase == p && c.Op.Stuck == "" } |
| + return func() bool { |
| + c := w.claim |
| + return c.Op.Kind == "relocating" && relocPhaseName(c.Op) == p && c.Op.Stuck == "" |
| + } |
| +} |
| + |
| +// relocPhaseName is the operation's phase as 1b-next names it: the |
| +// model's pinning with op.PinFailover set is production's starting. |
| +func relocPhaseName(op opState) string { |
| + if op.Phase == "pinning" && op.PinFailover { |
| + return "starting" |
| + } |
| + return op.Phase |
| } |
| |
| // liveIn counts region r's live boxes. |
| @@ -97,8 +111,8 @@ func TestL18TerminationAtEveryExactPhase(t *testing.T) { |
| for _, boxes := range []int{1, 2} { |
| for _, who := range []string{"source", "target"} { |
| for _, phase := range phases { |
| - if boxes == 1 && (who == "source" || phase == "starting" || phase == "releasing") { |
| - continue // a one-box region relocates only by failover, which starts at pinning with the source fenced |
| + if boxes == 1 && (who == "source" || phase == "releasing") { |
| + continue // a one-box region relocates only by failover, whose source is fenced at its start; its starting is PinFailover's wait (rev 7) |
| } |
| if phase == "probe-failed" && who == "source" { |
| continue // the source has released before any probe; its loss is the pinning case |
| @@ -117,9 +131,12 @@ func TestL18TerminationAtEveryExactPhase(t *testing.T) { |
| target = func() *box { return w.boxByID(w.claim.Op.Tgt) } |
| } else { |
| w.terminate(from) |
| - if !w.stepUntil(t, 400, "a target named", func() bool { c := w.claim; return c.Op.Kind == "relocating" && c.Op.Tgt >= 0 }) { |
| + if !w.stepUntil(t, 400, "the failover started with its target", func() bool { c := w.claim; return c.Op.Kind == "relocating" && c.Op.Tgt >= 0 }) { |
| w.mustHold(t) |
| } |
| + if c := w.claim; relocPhaseName(c.Op) != "starting" { |
| + t.Fatalf("the failover did not start in starting (PinFailover unset): %s", w.state()) |
| + } |
| w.boxByID(w.claim.Op.Tgt).FailApp = phase == "probe-failed" |
| target = func() *box { return w.boxByID(w.claim.Op.Tgt) } |
| } |
| @@ -128,9 +145,12 @@ func TestL18TerminationAtEveryExactPhase(t *testing.T) { |
| } |
| // immediately before the injection: the exact phase, and the |
| // region's live topology (no spare) |
| - if c := w.claim; c.Op.Phase != phase { |
| + if c := w.claim; relocPhaseName(c.Op) != phase { |
| t.Fatalf("not at %s: %s", phase, w.state()) |
| } |
| + if c := w.claim; phase == "pinning" && (c.Op.PinFailover || c.Op.Pin != nil && c.Op.Exec >= 0) { |
| + t.Fatalf("pinning is after PinFailover and before installing: %s", w.state()) |
| + } |
| if n := w.liveIn(0); n != boxes { |
| t.Fatalf("region 0 has %d live boxes before the injection, want %d", n, boxes) |
| } |
| @@ -469,20 +489,19 @@ func TestFailoverPinWaitsForTheAcknowledgement(t *testing.T) { |
| w.mustHold(t) |
| } |
| |
| -// TestConvertedMovePinsBeforeAcknowledgement (rev 6, Astra round 3 |
| -// finding 1): a move whose source is terminated before the flip becomes |
| -// a relocation at pinning (moves §7.10). It enters pinning directly, as |
| -// it does in 1b-next, where Worker.pinning selects and journals the pin |
| -// first and checks the acknowledgements only for ToInstalling: the pin is |
| -// journaled before the box that installs has acknowledged, and that box's |
| -// acknowledgement still comes before the installing CAS. Under rev 5's |
| -// wider guard the target acknowledged first, and this test fails. |
| -func TestConvertedMovePinsBeforeAcknowledgement(t *testing.T) { |
| - w := twoBoxWitness() |
| +// convertedMove serves the host on box 0 of a two-box witness world with |
| +// a spare box 30 in region 0, starts a move to region 1, and terminates |
| +// the source before the flip, so the regional worker converts the move |
| +// to a relocation at pinning with no target (moves §7.10). before runs |
| +// before the move starts. |
| +func (w *world) convertedMove(t *testing.T, before func()) *box { |
| src := w.boxes[0] |
| w.serve(src, 1, app0) |
| w.running(t, src) |
| w.published(t, src) |
| + if before != nil { |
| + before() |
| + } |
| w.moveTo(1) |
| if w.claim.Op.Kind != "moving" { |
| t.Fatalf("no move: %s", w.state()) |
| @@ -493,27 +512,147 @@ func TestConvertedMovePinsBeforeAcknowledgement(t *testing.T) { |
| return c.Op.Kind == "moving" && c.Op.Phase == "preparing" && contains(c.Regions, tgt.Region) |
| }, w.act(src, tgt)) |
| w.terminate(src) |
| - var acksAtPin []int |
| - if !w.stepUntil(t, 400, "the converted move's pin", func() bool { |
| + if !w.stepUntil(t, 400, "the move converted", func() bool { |
| c := w.claim |
| - if c.Op.Kind == "relocating" && c.Op.Pin != nil { |
| - acksAtPin = append([]int(nil), c.Op.Acks...) |
| - return true |
| - } |
| - return false |
| + return c.Op.Kind == "relocating" && c.Op.FenceBox == src.ID |
| }) { |
| w.mustHold(t) |
| } |
| - if c := w.claim; c.Op.FenceBox != src.ID || c.Op.PinFailover { |
| - t.Fatalf("not a converted move: %s", w.state()) |
| + if c := w.claim; c.Op.PinFailover || c.Op.Pin != nil { |
| + t.Fatalf("not a converted move at pinning without a pin: %s", w.state()) |
| + } |
| + return src |
| +} |
| + |
| +// TestConvertedMovePinsBeforeAcknowledgement (rev 6, Astra round 3 |
| +// finding 1; rev 7, round 4 finding 1): a converted move enters pinning |
| +// directly, as in 1b-next, and Worker.pinning names its target first, |
| +// then selects and journals the pin, and checks the acknowledgements only |
| +// for ToInstalling. The named target's controller does not run while the |
| +// region's other box works: the pin is journaled without the target's |
| +// acknowledgement, and installing waits for it. |
| +func TestConvertedMovePinsBeforeAcknowledgement(t *testing.T) { |
| + w := twoBoxWitness() |
| + spare := w.extraBox(0, 30) |
| + w.convertedMove(t, nil) |
| + if !w.stepUntil(t, 400, "the target named", func() bool { c := w.claim; return c.Op.Tgt >= 0 || c.Op.Pin != nil }) { |
| + w.mustHold(t) |
| + } |
| + c := w.claim |
| + if c.Op.Pin != nil { |
| + t.Fatalf("pinned before a target was named: %s", w.state()) |
| + } |
| + tgt := w.boxByID(c.Op.Tgt) |
| + worker := spare |
| + if tgt == spare { |
| + worker = w.boxes[3] |
| + } |
| + w.drive(failoverPinDelay+20, never, w.act(worker)) // the target's controller does not run |
| + c = w.claim |
| + if contains(c.Op.Acks, tgt.ID) { |
| + t.Fatalf("the target acknowledged without running") |
| + } |
| + if c.Op.Pin == nil { |
| + t.Fatalf("the converted move waited for an acknowledgement before its pin: %s", w.state()) |
| + } |
| + if c.Op.Phase != "pinning" || c.Op.Exec >= 0 { |
| + t.Fatalf("installing without the target's acknowledgement: %s", w.state()) |
| + } |
| + w.servingOn(t, tgt) |
| + w.finish(t) |
| + w.mustHold(t) |
| +} |
| + |
| +// TestConvertedMoveWaitsForATargetBeforeItsPin (reloc-name-before-pin; |
| +// rev 7, Astra round 4 finding 1): the source's newer files manifest is |
| +// in flight when the move starts and when the source is terminated. No |
| +// box of the region is eligible (the spare has no key), so the converted |
| +// move names no target and selects no pin; the manifest lands; the |
| +// spare becomes eligible, is named, and the pin names the newer |
| +// manifest. The mutation pins first, without the manifest. |
| +func TestConvertedMoveWaitsForATargetBeforeItsPin(t *testing.T) { |
| + w := twoBoxWitness() |
| + peer := w.boxes[3] |
| + var newer string |
| + w.convertedMove(t, func() { |
| + src, c := w.boxes[0], w.claim |
| + src.Files.Env++ // a newer files state: the admin token rotated |
| + newer = manifestKey(c.Holder.Location, c.Holder.Gen, src.pubSeq+1) |
| + w.holdKey(newer) |
| + w.publishFiles(src) |
| + w.hold = nil |
| + peer.HasKey = false // the region's other box may work, never take the host |
| + }) |
| + if newer == "" || w.boxes[0].pubSeq == 0 { |
| + t.Fatalf("the newer manifest was not issued") |
| + } |
| + w.drive(failoverPinDelay+20, never, w.act(peer)) |
| + if c := w.claim; c.Op.Pin != nil || c.Op.Tgt >= 0 { |
| + t.Fatalf("pinned or named with no eligible box: %s", w.state()) |
| + } |
| + if n := w.completeHeld(newer); n != 1 { |
| + t.Fatalf("held manifest PUTs completed: %d, want 1", n) |
| } |
| - var installer int = -1 |
| - if !w.stepUntil(t, 1500, "installing", func() bool { |
| + peer.HasKey = true |
| + if !w.reach(t, 400, "the pin", func() bool { return w.claim.Op.Pin != nil }, w.act(peer)) { |
| + w.mustHold(t) |
| + } |
| + if p := w.claim.Op.Pin; p.ManKey != newer { |
| + t.Fatalf("the pin names %s, want the manifest that landed while no box was eligible, %s", p.ManKey, newer) |
| + } |
| + w.servingOn(t, peer) |
| + w.finish(t) |
| + w.mustHold(t) |
| +} |
| + |
| +// TestFailoverStartWaitsForAdmission (reloc-failover-start-guard; rev |
| +// 7, Astra round 4 finding 3): with the app's admission closed, a |
| +// terminated serving executor's failover does not start (Worker.Start's |
| +// Admission.Register); once it reopens, it starts and serves on the |
| +// region's other box. |
| +func TestFailoverStartWaitsForAdmission(t *testing.T) { |
| + w := twoBoxWitness() |
| + from := w.boxes[0] |
| + w.serve(from, 1, app0) |
| + w.running(t, from) |
| + w.published(t, from) |
| + ad := w.admit[app0] |
| + ad.Closed = true |
| + w.terminate(from) |
| + w.drive(60, never, w.actAll(), w.recoveryPass) |
| + if c := w.claim; c.Op.Kind != "serving" || contains(c.Lost, from.ID) { |
| + t.Fatalf("a failover started while the app's admission was closed: %s", w.state()) |
| + } |
| + ad.Closed = false |
| + if to := w.failedOver(t, from); to == nil || to.Region != from.Region { |
| + t.Fatalf("no failover in the region: %s", w.state()) // the peer, or the group's replacement, whichever choose() prefers |
| + } |
| + w.finish(t) |
| + w.mustHold(t) |
| +} |
| + |
| +// TestFailoverStartWaitsForParkReplica (reloc-failover-start-guard; rev |
| +// 7, Astra round 4 finding 3): a park-replica chore is outstanding in the |
| +// region when the serving executor is terminated. RelocationStartGuard |
| +// refuses the failover's start until the chore is done; the source's own |
| +// park-local (the fenced source) would not have. |
| +func TestFailoverStartWaitsForParkReplica(t *testing.T) { |
| + w := twoBoxWitness() |
| + from := w.boxes[0] |
| + w.serve(from, 1, app0) |
| + w.running(t, from) |
| + w.published(t, from) |
| + w.claim.Obl = append(w.claim.Obl, w.scoped(obligation{ID: "pr", Op: "old-op", Kind: "park-replica", Region: 0, Box: -1, Prefix: "moves/" + host + "/", Quiet: w.now + 20}, w.now, 1)) |
| + w.claim.Obl = append(w.claim.Obl, obligation{ID: "pl", Op: "old-op", Kind: "park-local", Box: from.ID, Region: -1, Ret: "old-op"}) |
| + from.DeleteOnTermination = false // the fence keeps the park-local open |
| + w.terminate(from) |
| + if !w.stepUntil(t, 400, "the failover's start", func() bool { |
| c := w.claim |
| - if c.Op.Kind == "relocating" && c.Op.Phase == "installing" { |
| - installer = c.Op.Exec |
| - if !acked(c) { |
| - t.Fatalf("installing without the acknowledgements: %s", w.state()) |
| + if c.Op.Kind == "relocating" { |
| + for _, o := range c.Obl { |
| + if o.Kind == "park-replica" && o.Region == 0 { |
| + t.Fatalf("the failover started with a park-replica outstanding in its region: %s", w.state()) |
| + } |
| } |
| return true |
| } |
| @@ -521,14 +660,56 @@ func TestConvertedMovePinsBeforeAcknowledgement(t *testing.T) { |
| }) { |
| w.mustHold(t) |
| } |
| - if contains(acksAtPin, installer) { |
| - t.Fatalf("box %d acknowledged before the pin: a converted move waited for PinFailover's barrier", installer) |
| + if !w.owesLocal(w.claim, from) { |
| + t.Fatalf("the fenced source's park-local was not kept open: %s", w.state()) |
| + } |
| + if to := w.failedOver(t, from); to == nil || to.Region != from.Region { |
| + t.Fatalf("no failover in the region: %s", w.state()) |
| } |
| - w.servingOn(t, w.boxByID(installer)) |
| w.finish(t) |
| w.mustHold(t) |
| } |
| |
| +// TestFailoverTargetLossAroundPinFailover (rev 7, Astra round 4 finding |
| +// 2): an initial failover's target is terminated while PinFailover is |
| +// pending (production's starting) and, in a second run, after it has |
| +// cleared and before installing (pinning). In a two-box region with a |
| +// spare, the target is replaced in the same phase and the host serves. |
| +func TestFailoverTargetLossAroundPinFailover(t *testing.T) { |
| + for _, phase := range []string{"starting", "pinning"} { |
| + t.Run(phase, func(t *testing.T) { |
| + w := twoBoxWitness() |
| + from := w.boxes[0] |
| + w.extraBox(0, 30) |
| + w.serve(from, 1, app0) |
| + w.running(t, from) |
| + w.published(t, from) |
| + w.terminate(from) |
| + if !w.stepUntil(t, 400, "the failover exactly at "+phase, w.atPhase(phase)) { |
| + w.mustHold(t) |
| + } |
| + c := w.claim |
| + if (phase == "starting") != c.Op.PinFailover || c.Op.Exec >= 0 || c.Op.Tgt < 0 { |
| + t.Fatalf("not at %s: %s", phase, w.state()) |
| + } |
| + victim := w.boxByID(c.Op.Tgt) |
| + w.terminate(victim) |
| + if !w.stepUntil(t, 400, "the target replaced", func() bool { |
| + c := w.claim |
| + return contains(c.Lost, victim.ID) && c.Op.Tgt >= 0 && c.Op.Tgt != victim.ID |
| + }) { |
| + w.mustHold(t) |
| + } |
| + if got := relocPhaseName(w.claim.Op); got != phase { |
| + t.Fatalf("the replacement left %s for %s: %s", phase, got, w.state()) |
| + } |
| + w.servingOn(t, nil) |
| + w.finish(t) |
| + w.mustHold(t) |
| + }) |
| + } |
| +} |
| + |
| // TestPinNeverShortOfTheRelease (reloc-pin-reaches-release): the listing |
| // lags behind the journaled release (its newest file is not yet visible); |
| // no pin is journaled short of the release's final_txid. |
amadan