| 1 | package activator |
| 2 | |
| 3 | import ( |
| 4 | "errors" |
| 5 | "fmt" |
| 6 | "testing" |
| 7 | "time" |
| 8 | ) |
| 9 | |
| 10 | // TestFenceAndStopAsMeetsOnlyItsOwnFence: an owned fence is the caller's |
| 11 | // again only when it holds at the same generation with the same owner. |
| 12 | // Any other fence (a plain one at a lower, equal or higher generation, or |
| 13 | // another owner's at the same generation) refuses with ErrFenceHeld and |
| 14 | // stays exactly as it was, with nothing stopped. |
| 15 | func TestFenceAndStopAsMeetsOnlyItsOwnFence(t *testing.T) { |
| 16 | t.Parallel() |
| 17 | t.Run("own fence, re-collected", func(t *testing.T) { |
| 18 | a, e, _, _, _ := maintenanceAwake(t) |
| 19 | proof, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1") |
| 20 | if err != nil || !proof.Matches(e.host, 12) { |
| 21 | t.Fatalf("first install: %v", err) |
| 22 | } |
| 23 | again, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1") |
| 24 | if err != nil || !again.Matches(e.host, 12) { |
| 25 | t.Fatalf("the owner's repeat: %v", err) |
| 26 | } |
| 27 | if g := a.FenceGeneration(e.host); g != 12 { |
| 28 | t.Fatalf("generation %d, want 12", g) |
| 29 | } |
| 30 | }) |
| 31 | type held struct { |
| 32 | gen uint64 |
| 33 | owner string |
| 34 | } |
| 35 | for _, h := range []held{{5, ""}, {12, ""}, {20, ""}, {12, "owner-2"}, {5, "owner-1"}} { |
| 36 | t.Run(fmt.Sprintf("held %d by %q", h.gen, h.owner), func(t *testing.T) { |
| 37 | a, e, main, _, _ := maintenanceAwake(t) |
| 38 | a.mu.Lock() |
| 39 | a.fences = map[string]uint64{e.host: h.gen} |
| 40 | if h.owner != "" { |
| 41 | a.fenceOwners = map[string]string{e.host: h.owner} |
| 42 | } |
| 43 | a.mu.Unlock() |
| 44 | if _, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1"); !errors.Is(err, ErrFenceHeld) { |
| 45 | t.Fatalf("err = %v, want ErrFenceHeld", err) |
| 46 | } |
| 47 | a.mu.Lock() |
| 48 | gen, owner := a.fences[e.host], a.fenceOwners[e.host] |
| 49 | a.mu.Unlock() |
| 50 | if gen != h.gen || owner != h.owner { |
| 51 | t.Fatalf("held fence is %d by %q, want %d by %q", gen, owner, h.gen, h.owner) |
| 52 | } |
| 53 | if main.calls.Load() != 0 || !a.Owns(e.host) { |
| 54 | t.Fatal("a refused owned fence stopped the host") |
| 55 | } |
| 56 | }) |
| 57 | } |
| 58 | t.Run("an empty owner", func(t *testing.T) { |
| 59 | a, e, _, _, _ := maintenanceAwake(t) |
| 60 | if _, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, ""); !errors.Is(err, ErrFenceGeneration) || a.Fenced(e.host) { |
| 61 | t.Fatalf("err = %v, fenced %v", err, a.Fenced(e.host)) |
| 62 | } |
| 63 | }) |
| 64 | } |
| 65 | |
| 66 | // TestAPlainFenceTakesOwnershipAway: FenceAndStop at the same generation |
| 67 | // over an owned fence leaves a fence that is no longer the owner's, so |
| 68 | // the owner can neither re-collect under it nor lift it. |
| 69 | func TestAPlainFenceTakesOwnershipAway(t *testing.T) { |
| 70 | a, e, _, _, _ := maintenanceAwake(t) |
| 71 | if _, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1"); err != nil { |
| 72 | t.Fatal(err) |
| 73 | } |
| 74 | if _, err := a.FenceAndStop(maintenanceContext(t, time.Second), e.host, 12); !errors.Is(err, ErrStopUnproven) && err != nil { |
| 75 | t.Fatal(err) |
| 76 | } |
| 77 | if _, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1"); !errors.Is(err, ErrFenceHeld) { |
| 78 | t.Fatalf("err = %v, want ErrFenceHeld", err) |
| 79 | } |
| 80 | if a.LiftFenceAs(e.host, 12, "owner-1") { |
| 81 | t.Fatal("the owner lifted a fence it no longer owns") |
| 82 | } |
| 83 | if !a.Fenced(e.host) { |
| 84 | t.Fatal("the plain fence was lifted") |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | // TestLiftFenceAsLiftsOnlyTheOwnersFence: the lift names the generation |
| 89 | // and the owner, and a plain LiftFence also clears the owner. |
| 90 | func TestLiftFenceAsLiftsOnlyTheOwnersFence(t *testing.T) { |
| 91 | a, e, _, _, _ := maintenanceAwake(t) |
| 92 | if _, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 12, "owner-1"); err != nil { |
| 93 | t.Fatal(err) |
| 94 | } |
| 95 | if a.LiftFenceAs(e.host, 12, "owner-2") || a.LiftFenceAs(e.host, 11, "owner-1") || a.LiftFenceAs(e.host, 12, "") { |
| 96 | t.Fatal("lifted with the wrong owner or generation") |
| 97 | } |
| 98 | if !a.LiftFenceAs(e.host, 12, "owner-1") || a.Fenced(e.host) { |
| 99 | t.Fatal("the owner's lift did not lift its fence") |
| 100 | } |
| 101 | if _, err := a.FenceAndStopAs(maintenanceContext(t, time.Second), e.host, 13, "owner-1"); err != nil && !errors.Is(err, ErrStopUnproven) { |
| 102 | t.Fatal(err) |
| 103 | } |
| 104 | if !a.LiftFence(e.host, 13) { |
| 105 | t.Fatal("plain lift") |
| 106 | } |
| 107 | a.mu.Lock() |
| 108 | _, left := a.fenceOwners[e.host] |
| 109 | a.mu.Unlock() |
| 110 | if left { |
| 111 | t.Fatal("a plain lift left the owner behind") |
| 112 | } |
| 113 | } |
| 114 | |
| 115 | // TestAnOwnedCallNeverTakesBackAFenceReplacedWhileItWaited (fix round 1, |
| 116 | // M1; ruling 75): FenceAndStopAs installs its owned fence and waits for |
| 117 | // the stop. Meanwhile a plain FenceAndStop installs at the same generation, |
| 118 | // or a plain LiftFence lifts it. When the stop completes the owned call |
| 119 | // answers ErrStopUnproven and leaves the fence as the other caller left |
| 120 | // it: still plain and unowned, or still lifted, never the owner's again. |
| 121 | func TestAnOwnedCallNeverTakesBackAFenceReplacedWhileItWaited(t *testing.T) { |
| 122 | t.Parallel() |
| 123 | for _, meanwhile := range []string{"a plain fence", "a plain lift"} { |
| 124 | t.Run(meanwhile, func(t *testing.T) { |
| 125 | a, e, _, _, _ := maintenanceAwake(t) |
| 126 | stopped := make(chan struct{}) |
| 127 | a.mu.Lock() |
| 128 | e.state, e.stopped = actStopping, stopped |
| 129 | a.mu.Unlock() |
| 130 | type answer struct { |
| 131 | proof *StopEvidence |
| 132 | err error |
| 133 | } |
| 134 | done := make(chan answer, 1) |
| 135 | go func() { |
| 136 | proof, err := a.FenceAndStopAs(maintenanceContext(t, 30*time.Second), e.host, 12, "owner-1") |
| 137 | done <- answer{proof, err} |
| 138 | }() |
| 139 | for deadline := time.Now().Add(5 * time.Second); ; { |
| 140 | a.mu.Lock() |
| 141 | installed := a.fences[e.host] == 12 && a.fenceOwners[e.host] == "owner-1" |
| 142 | a.mu.Unlock() |
| 143 | if installed { |
| 144 | break |
| 145 | } |
| 146 | if time.Now().After(deadline) { |
| 147 | t.Fatal("the owned call never installed its fence") |
| 148 | } |
| 149 | time.Sleep(time.Millisecond) |
| 150 | } |
| 151 | switch meanwhile { |
| 152 | case "a plain fence": |
| 153 | // The stop is still in progress, so the plain call installs and |
| 154 | // then waits out its own deadline: ErrStopUnproven. |
| 155 | if _, err := a.FenceAndStop(maintenanceContext(t, 500*time.Millisecond), e.host, 12); !errors.Is(err, ErrStopUnproven) { |
| 156 | t.Fatalf("the plain fence: %v", err) |
| 157 | } |
| 158 | case "a plain lift": |
| 159 | if !a.LiftFence(e.host, 12) { |
| 160 | t.Fatal("the plain lift lifted nothing") |
| 161 | } |
| 162 | } |
| 163 | a.mu.Lock() |
| 164 | delete(a.entries, e.host) |
| 165 | a.mu.Unlock() |
| 166 | close(stopped) |
| 167 | var got answer |
| 168 | select { |
| 169 | case got = <-done: |
| 170 | case <-time.After(30 * time.Second): |
| 171 | t.Fatal("the owned call never answered after the stop completed") |
| 172 | } |
| 173 | if !errors.Is(got.err, ErrStopUnproven) || got.proof != nil { |
| 174 | t.Fatalf("the owned call after its fence was replaced: %v %v", got.proof, got.err) |
| 175 | } |
| 176 | a.mu.Lock() |
| 177 | gen := a.fences[e.host] |
| 178 | owner, owned := a.fenceOwners[e.host] |
| 179 | a.mu.Unlock() |
| 180 | want := uint64(12) |
| 181 | if meanwhile == "a plain lift" { |
| 182 | want = 0 |
| 183 | } |
| 184 | if gen != want || owned { |
| 185 | t.Fatalf("the fence is %d owned by %q (%v), want %d unowned", gen, owner, owned, want) |
| 186 | } |
| 187 | }) |
| 188 | } |
| 189 | } |
| 190 | |
amadan