| 1 | package rollout |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "slices" |
| 6 | "sync" |
| 7 | "testing" |
| 8 | "time" |
| 9 | |
| 10 | "github.com/carlosframework/platform/internal/activator" |
| 11 | "github.com/carlosframework/platform/internal/appconfig" |
| 12 | ) |
| 13 | |
| 14 | // N6, Astra's sequence: A's good write was decided before hold B ran on the |
| 15 | // host; B's marker is newer, and A's write must not clear it. |
| 16 | func TestAnOlderGoodWriteNeverClearsANewerMarker(t *testing.T) { |
| 17 | b := &FailureBook{} |
| 18 | snapA := b.Snapshot() |
| 19 | if err := b.MarkHoldRan("h", HoldRan{Account: "bab", App: "blog", Build: "v2"}); err != nil { |
| 20 | t.Fatal(err) |
| 21 | } |
| 22 | if cleared, _ := b.ClearHoldRanBefore("h", snapA); cleared { |
| 23 | t.Fatal("a write decided before the marker cleared it") |
| 24 | } |
| 25 | if cleared, _ := b.ClearHoldRanBefore("h", b.Snapshot()); !cleared { |
| 26 | t.Fatal("a write decided after the marker must clear it") |
| 27 | } |
| 28 | } |
| 29 | |
| 30 | func TestAZeroSnapshotClearsNothing(t *testing.T) { |
| 31 | b := &FailureBook{} |
| 32 | _ = b.MarkHoldRan("h", HoldRan{Account: "bab", App: "blog", Build: "v2"}) |
| 33 | if cleared, _ := b.ClearHoldRanBefore("h", 0); cleared { |
| 34 | t.Fatal("no snapshot must clear nothing") |
| 35 | } |
| 36 | } |
| 37 | |
| 38 | // Plan review finding 4: a marker belongs to one execution, not to its |
| 39 | // build. B's marker (execution a1) stands before C's snapshot; B then runs |
| 40 | // again (a2) while C's good write is delayed. The second execution renews |
| 41 | // the marker's sequence, so C's write clears nothing. A repeated report of |
| 42 | // one execution changes nothing. |
| 43 | func TestAnotherExecutionOfTheMarkedBuildRenewsTheMarker(t *testing.T) { |
| 44 | now := t0 |
| 45 | b := &FailureBook{now: func() time.Time { return now }} |
| 46 | if err := b.MarkHoldRan("h", HoldRan{Account: "bab", App: "blog", Build: "v2", Exec: "a1", At: t0}); err != nil { |
| 47 | t.Fatal(err) |
| 48 | } |
| 49 | snapC := b.Snapshot() |
| 50 | second := t0.Add(time.Minute) |
| 51 | if err := b.MarkHoldRan("h", HoldRan{Account: "bab", App: "blog", Build: "v2", Exec: "a2", At: second}); err != nil { |
| 52 | t.Fatal(err) |
| 53 | } |
| 54 | got, _ := b.HoldRanOn("h") |
| 55 | if got.Exec != "a2" || got.Seq < snapC || !got.At.Equal(second) { |
| 56 | t.Fatalf("marker %+v after a second execution of v2: want a2's, sequence at least %d, stamped %v", got, snapC, second) |
| 57 | } |
| 58 | if cleared, _ := b.ClearHoldRanBefore("h", snapC); cleared { |
| 59 | t.Fatal("C's good write cleared the marker of an execution that began after C's snapshot") |
| 60 | } |
| 61 | // The same execution reported again keeps its sequence and its stamp. |
| 62 | _ = b.MarkHoldRan("h", HoldRan{Account: "bab", App: "blog", Build: "v2", Exec: "a2", At: second.Add(time.Minute)}) |
| 63 | if again, _ := b.HoldRanOn("h"); again != got { |
| 64 | t.Fatalf("marker %+v after a repeated report of a2, want it unchanged at %+v", again, got) |
| 65 | } |
| 66 | // A late report of an earlier execution of the same build never moves |
| 67 | // the stamp back: a rollback signed between the two runs predates a2. |
| 68 | _ = b.MarkHoldRan("h", HoldRan{Account: "bab", App: "blog", Build: "v2", Exec: "a0", At: t0.Add(-time.Minute)}) |
| 69 | late, _ := b.HoldRanOn("h") |
| 70 | if !late.At.Equal(second) || late.Seq <= got.Seq { |
| 71 | t.Fatalf("marker %+v after a late report of an earlier run: want the stamp kept at %v and a newer sequence", late, second) |
| 72 | } |
| 73 | // The same for another build's older execution (Task 4 fix round 1): |
| 74 | // the marker names the newest report's build, never an older stamp. |
| 75 | _ = b.MarkHoldRan("h", HoldRan{Account: "bab", App: "blog", Build: "v3", Exec: "a9", At: t0}) |
| 76 | if other, _ := b.HoldRanOn("h"); other.Build != "v3" || !other.At.Equal(second) || other.Seq <= late.Seq { |
| 77 | t.Fatalf("marker %+v after a late report of another build's earlier run: want v3's, stamped %v, newer sequence", other, second) |
| 78 | } |
| 79 | } |
| 80 | |
| 81 | // Retry ordering: a failure of an attempt that began before the host's retry |
| 82 | // never takes the retry back. |
| 83 | func TestAPreRetryFailureLeavesTheRetryStanding(t *testing.T) { |
| 84 | b := &FailureBook{} |
| 85 | before := b.Snapshot() |
| 86 | b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "s", Seq: before}) |
| 87 | if ok, _ := b.RetryHost("bab", "blog", "v2", "h"); !ok { |
| 88 | t.Fatal("retry") |
| 89 | } |
| 90 | if res, _ := b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h", Seq: before, HeldDown: []string{"h"}}); res != PreRetry { |
| 91 | t.Fatalf("Record = %v, want PreRetry", res) |
| 92 | } |
| 93 | if b.Blocks("bab", "blog", "v2", "h", false) || b.HeldDown("bab", "blog", "v2", "h") { |
| 94 | t.Fatal("a pre-retry failure undid the retry") |
| 95 | } |
| 96 | b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h", Seq: b.Snapshot()}) |
| 97 | if !b.Blocks("bab", "blog", "v2", "h", false) { |
| 98 | t.Fatal("a failure after the retry must stand") |
| 99 | } |
| 100 | } |
| 101 | |
| 102 | func TestAPreRetryFailureNeverRecreatesAClearedRecord(t *testing.T) { |
| 103 | b := &FailureBook{} |
| 104 | before := b.Snapshot() |
| 105 | b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "s", Seq: before}) |
| 106 | if ok, _ := b.ClearOnPointer("bab", "blog", "v2", time.Now().Add(time.Hour)); !ok { |
| 107 | t.Fatal("clear") |
| 108 | } |
| 109 | b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "t", Seq: before, MarkHoldRan: true, Mode: "hold"}) |
| 110 | if _, held := b.Lookup("bab", "blog", "v2"); held { |
| 111 | t.Fatal("a pre-retry failure recreated the record") |
| 112 | } |
| 113 | if _, marked := b.HoldRanOn("t"); !marked { |
| 114 | t.Fatal("the marker is a fact about t's database and must still be set") |
| 115 | } |
| 116 | } |
| 117 | |
| 118 | // A pre-retry hold-down is a hold build that ran against the instance's |
| 119 | // database, like a requested marker: the retry keeps the hold-down off, and |
| 120 | // the instance keeps the marker (no automatic fallback onto it, spec §6.4; |
| 121 | // Task 4 fix round 1), with the execution's identity and start. |
| 122 | func TestAPreRetryHoldDownLeavesTheMarker(t *testing.T) { |
| 123 | b := &FailureBook{} |
| 124 | before := b.Snapshot() |
| 125 | b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "s", Seq: before}) |
| 126 | b.RetryHost("bab", "blog", "v2", "h") |
| 127 | started := t0.Add(-time.Minute) |
| 128 | b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h", Seq: before, Mode: "hold", HeldDown: []string{"h"}, |
| 129 | HoldRanAt: started, HoldRanExec: "a7"}) |
| 130 | if b.HeldDown("bab", "blog", "v2", "h") { |
| 131 | t.Fatal("a pre-retry failure held the retried host down") |
| 132 | } |
| 133 | got, ok := b.HoldRanOn("h") |
| 134 | if !ok || got.Build != "v2" || got.Exec != "a7" || !got.At.Equal(started) { |
| 135 | t.Fatalf("marker %+v, %v: want v2's from execution a7, stamped %v", got, ok, started) |
| 136 | } |
| 137 | } |
| 138 | |
| 139 | // Task 4 fix round 1, the reviewer's stranding sequence: a retry lets b's |
| 140 | // canary through the halt; a second retry arrives while that canary runs; |
| 141 | // the canary fails. Its failure began after the first retry, which is the |
| 142 | // one it spends: it is recorded and b is held again, and a later retry runs |
| 143 | // another attempt. A second retry of a host still retried notes no clear, |
| 144 | // or the canary's failure would be stale, b would stay retried behind the |
| 145 | // halt with its allowance spent, and no retry could free it. |
| 146 | func TestASecondRetryDuringTheCanaryNeverStrandsTheHost(t *testing.T) { |
| 147 | h := newHarness(t, "a", "b") |
| 148 | h.stage("v2") |
| 149 | h.failing.Store(true) |
| 150 | h.act.outcome = func(a activator.Attempt) activator.SwapOutcome { |
| 151 | if a.Build == "v2" && h.failing.Load() { |
| 152 | return activator.SwapFailed |
| 153 | } |
| 154 | return activator.SwapDone |
| 155 | } |
| 156 | h.install("v2", "a", "b") |
| 157 | h.pass() |
| 158 | waitFor(t, "a failure record", func() bool { _, ok := h.book.Lookup("bab", "blog", "v2"); return ok }) |
| 159 | gate := h.act.hold("b") |
| 160 | if !h.retry("b", "v2") { |
| 161 | t.Fatal("the first retry cleared nothing") |
| 162 | } |
| 163 | waitFor(t, "b's retry canary", func() bool { |
| 164 | _, started, _ := h.act.snapshot() |
| 165 | return slices.Contains(started, "b:v2") |
| 166 | }) |
| 167 | h.retry("b", "v2") // the second retry, while the canary runs |
| 168 | close(gate) |
| 169 | waitFor(t, "b's canary to fail", func() bool { return !h.m.attemptRunning("b") && h.book.Blocks("bab", "blog", "v2", "b", false) }) |
| 170 | |
| 171 | h.failing.Store(false) |
| 172 | if !h.retry("b", "v2") { |
| 173 | t.Fatal("a retry after the canary's failure cleared nothing: b is stranded") |
| 174 | } |
| 175 | waitFor(t, "b on v2", func() bool { return h.servingAll("v2", "b") }) |
| 176 | _, started, _ := h.act.snapshot() |
| 177 | n := 0 |
| 178 | for _, st := range started { |
| 179 | if st == "b:v2" { |
| 180 | n++ |
| 181 | } |
| 182 | } |
| 183 | if n != 2 { |
| 184 | t.Fatalf("attempts %v: want b's canary and the attempt after the third retry", started) |
| 185 | } |
| 186 | } |
| 187 | |
| 188 | // Task 4 fix round 1: a planned wake's snapshot is taken at the plan's end. |
| 189 | // A retry that lands during the planner's slow reads (here the policy load) |
| 190 | // is what let the plan boot the retried build, so the build's failure in |
| 191 | // that wake is a real one: recorded, and in hold mode a hold-down. |
| 192 | func TestARetryDuringThePlanLeavesTheWakesFailureStanding(t *testing.T) { |
| 193 | ctx := context.Background() |
| 194 | h := newHarness(t, "w") |
| 195 | h.swap = appconfig.SwapHold |
| 196 | h.stage("v2") |
| 197 | h.install("v2", "w") |
| 198 | h.sleepHost("w") |
| 199 | h.pass() |
| 200 | h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "s", PromotedAt: t0, At: t0}) |
| 201 | h.wakeHost("w") |
| 202 | h.advance(time.Minute) // the policy cache is stale: the plan loads it |
| 203 | var once sync.Once |
| 204 | h.mu.Lock() |
| 205 | h.policyHook = func() { |
| 206 | once.Do(func() { |
| 207 | if !h.retry("w", "v2") { |
| 208 | t.Error("the retry during the plan cleared nothing") |
| 209 | } |
| 210 | }) |
| 211 | } |
| 212 | h.mu.Unlock() |
| 213 | p := h.m.Plan(ctx, h.rows.get("w")) |
| 214 | if !p.FirstWake || p.Boot.Version != "v2" || p.Mode != activator.ModeHold { |
| 215 | t.Fatalf("plan %+v: want a hold first wake of the retried v2", p) |
| 216 | } |
| 217 | h.m.OnGen(activator.GenEvent{Kind: activator.GenFailed, Route: h.rows.get("w"), Failed: p.Boot, Mode: p.Mode, |
| 218 | FirstWake: true, Reason: "health check failed", Token: p.Token, ConfigRev: p.ConfigRev}) |
| 219 | if !h.book.Blocks("bab", "blog", "v2", "w", false) || !h.book.HeldDown("bab", "blog", "v2", "w") { |
| 220 | t.Fatal("the failure of a wake the retry let through was taken as pre-retry") |
| 221 | } |
| 222 | } |
| 223 | |
| 224 | // What Record reports, which failCohort acts on (plan review finding 3). |
| 225 | func TestRecordReportsWhatItDid(t *testing.T) { |
| 226 | b := &FailureBook{} |
| 227 | if res, _ := b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a"}); res != Recorded { |
| 228 | t.Fatalf("first failure: %v, want Recorded", res) |
| 229 | } |
| 230 | if res, _ := b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "b", HeldDown: []string{"b"}}); res != Merged { |
| 231 | t.Fatalf("a new hold-down: %v, want Merged", res) |
| 232 | } |
| 233 | if res, _ := b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "b", HeldDown: []string{"b"}}); res != Unchanged { |
| 234 | t.Fatalf("the same failure again: %v, want Unchanged", res) |
| 235 | } |
| 236 | if res, _ := b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "c", Seq: b.Snapshot()}); res == PreRetry { |
| 237 | t.Fatal("a failure with no retry before it was taken as pre-retry") |
| 238 | } |
| 239 | } |
| 240 | |
| 241 | // A retry clear goes with its build's record when no route installs the |
| 242 | // build: Prune drops both. |
| 243 | func TestPruneDropsTheRetryClearsOfBuildsNoRouteInstalls(t *testing.T) { |
| 244 | b := &FailureBook{} |
| 245 | before := b.Snapshot() |
| 246 | b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "s"}) |
| 247 | b.RetryHost("bab", "blog", "v2", "h") |
| 248 | if err := b.Prune(func(string, string, string) bool { return false }); err != nil { |
| 249 | t.Fatal(err) |
| 250 | } |
| 251 | if len(b.cleared) != 0 { |
| 252 | t.Fatalf("clears %v survived the prune of their build", b.cleared) |
| 253 | } |
| 254 | if res, _ := b.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "h", Seq: before}); res != Recorded { |
| 255 | t.Fatalf("Record = %v after the prune, want Recorded", res) |
| 256 | } |
| 257 | } |
| 258 | |
| 259 | func TestTheSequenceSurvivesAReopen(t *testing.T) { |
| 260 | dd := t.TempDir() |
| 261 | b, _ := OpenFailureBook(dd) |
| 262 | _ = b.MarkHoldRan("h", HoldRan{Account: "bab", App: "blog", Build: "v2"}) |
| 263 | b2, err := OpenFailureBook(dd) |
| 264 | if err != nil { |
| 265 | t.Fatal(err) |
| 266 | } |
| 267 | if cleared, _ := b2.ClearHoldRanBefore("h", b2.Snapshot()); !cleared { |
| 268 | t.Fatal("after a reopen a fresh snapshot must exceed the old marker") |
| 269 | } |
| 270 | } |
| 271 | |
| 272 | // N6 through the Manager: h's first wake of v1 is planned (its snapshot |
| 273 | // taken) and is observed; meanwhile a scheduled hold attempt onto v2 runs |
| 274 | // on h and is superseded, which marks h. v1's settle then arrives and |
| 275 | // writes v1 as h's good build: the write was decided before v2 ran, so v2's |
| 276 | // marker stands. |
| 277 | func TestASettleDecidedBeforeAHoldRanKeepsItsMarker(t *testing.T) { |
| 278 | ctx := context.Background() |
| 279 | h := newHarness(t, "h") |
| 280 | r := h.rows.get("h") |
| 281 | r.GoodVersion, r.GoodDigest = "", "" // v1 has not passed here yet |
| 282 | h.rows.put(r) |
| 283 | h.swap = appconfig.SwapHold |
| 284 | h.stage("v2") |
| 285 | h.act.mode = activator.ModeHold |
| 286 | h.act.executed = func(a activator.Attempt, _ activator.SwapOutcome) bool { return a.Build == "v2" } |
| 287 | |
| 288 | p := h.m.Plan(ctx, h.rows.get("h")) |
| 289 | if !p.FirstWake || p.Boot.Version != "v1" { |
| 290 | t.Fatalf("plan %+v: want a first wake of v1", p) |
| 291 | } |
| 292 | g, _ := h.act.Current("h") |
| 293 | h.m.OnGen(activator.GenEvent{Kind: activator.GenHealthy, Route: h.rows.get("h"), Gen: g, Mode: p.Mode, FirstWake: true, Token: p.Token}) |
| 294 | // Since Task 5 the wake's intent, which stands until its settle, refuses |
| 295 | // an adoption of h (ErrIntentStands), so this ordering needs the intent |
| 296 | // gone. It is removed by hand: the marker rule must hold whatever lets a |
| 297 | // hold run between a plan and its settle. |
| 298 | if err := h.book.End("h", h.m.wakeExecID(p.Token)); err != nil { |
| 299 | t.Fatal(err) |
| 300 | } |
| 301 | |
| 302 | gate := h.act.hold("h") |
| 303 | h.install("v2", "h") |
| 304 | h.pass() |
| 305 | waitFor(t, "h's hold attempt onto v2", func() bool { |
| 306 | _, started, _ := h.act.snapshot() |
| 307 | return slices.Contains(started, "h:v2") |
| 308 | }) |
| 309 | h.install("v3", "h") // v3 is not staged: its attempt is refused, unrun |
| 310 | h.pass() |
| 311 | close(gate) |
| 312 | waitFor(t, "v2's marker", func() bool { m, ok := h.book.HoldRanOn("h"); return ok && m.Build == "v2" }) |
| 313 | |
| 314 | h.m.OnGen(activator.GenEvent{Kind: activator.GenSettled, Route: h.rows.get("h"), Gen: g, Mode: p.Mode, FirstWake: true, Token: p.Token}) |
| 315 | if r := h.rows.get("h"); r.GoodVersion != "v1" { |
| 316 | t.Fatalf("good build %q after v1's settle, want v1", r.GoodVersion) |
| 317 | } |
| 318 | if m, ok := h.book.HoldRanOn("h"); !ok || m.Build != "v2" { |
| 319 | t.Fatalf("marker %+v, %v after v1's settle: want v2's, set after v1's wake was planned", m, ok) |
| 320 | } |
| 321 | } |
| 322 | |
| 323 | // Plan review finding 3: an outcome that began before a retry is stale, and |
| 324 | // none of the failure transition runs for it. After `carlos instances retry` |
| 325 | // of the failing host, or a `carlos promote --retry` pointer that cleared |
| 326 | // the record, a delayed failure of an attempt or of a planned wake that |
| 327 | // began before it leaves the cohort unhalted, supersedes no running attempt |
| 328 | // (y's), and leaves the retried host free: its queued adoption is admitted, |
| 329 | // or its next wake plans the retried build. |
| 330 | func TestAPreRetryOutcomeRunsNoFailureTransition(t *testing.T) { |
| 331 | for _, retry := range []string{"host", "pointer"} { |
| 332 | for _, kind := range []string{"attempt", "wake"} { |
| 333 | t.Run(retry+" retry, "+kind, func(t *testing.T) { |
| 334 | preRetryOutcome(t, retry, kind) |
| 335 | }) |
| 336 | } |
| 337 | } |
| 338 | } |
| 339 | |
| 340 | func preRetryOutcome(t *testing.T, retry, kind string) { |
| 341 | ctx := context.Background() |
| 342 | k := appKey{"bab", "blog"} |
| 343 | h := newHarness(t, "h", "y", "w") |
| 344 | h.concurrency = 2 |
| 345 | h.swap = appconfig.SwapHold |
| 346 | h.stage("v2") |
| 347 | h.install("v2", "h", "y", "w") |
| 348 | s := h.m.scheduler(k) |
| 349 | s.mu.Lock() |
| 350 | // The cohort's canary passed on another host already, so h and y run |
| 351 | // side by side. |
| 352 | *s.q.cohortFor(cohortKey("v2", classAdopt)) = cohort{canary: "x", settled: true} |
| 353 | s.mu.Unlock() |
| 354 | gateH, gateY := h.act.hold("h"), h.act.hold("y") |
| 355 | h.sleepHost("w") |
| 356 | h.pass() |
| 357 | waitFor(t, "h's and y's attempts onto v2", func() bool { |
| 358 | _, started, _ := h.act.snapshot() |
| 359 | return slices.Contains(started, "h:v2") && slices.Contains(started, "y:v2") |
| 360 | }) |
| 361 | host := "h" |
| 362 | var plan activator.WakePlan |
| 363 | if kind == "wake" { |
| 364 | host = "w" |
| 365 | h.wakeHost("w") |
| 366 | plan = h.m.Plan(ctx, h.rows.get("w")) |
| 367 | if !plan.FirstWake || plan.Boot.Version != "v2" { |
| 368 | t.Fatalf("w's plan %+v: want a first wake of v2", plan) |
| 369 | } |
| 370 | } |
| 371 | |
| 372 | // A sibling's failure of v2, written straight into the book: no halt |
| 373 | // stands, so a halt after this point is the stale outcome's. |
| 374 | h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "s", PromotedAt: t0, At: t0}) |
| 375 | switch retry { |
| 376 | case "host": |
| 377 | if !h.retry(host, "v2") { |
| 378 | t.Fatalf("RetryHost(%s) cleared nothing", host) |
| 379 | } |
| 380 | case "pointer": |
| 381 | h.advance(time.Minute) |
| 382 | h.pointer("stable", "v2", h.clock()) |
| 383 | h.pass() |
| 384 | if _, held := h.book.Lookup("bab", "blog", "v2"); held { |
| 385 | t.Fatal("the retry pointer left v2's record standing") |
| 386 | } |
| 387 | } |
| 388 | s.mu.Lock() |
| 389 | revY := s.q.newest("y") |
| 390 | s.mu.Unlock() |
| 391 | |
| 392 | // The outcome that began before the retry arrives now. |
| 393 | if kind == "attempt" { |
| 394 | h.act.mu.Lock() |
| 395 | h.act.outcome = func(a activator.Attempt) activator.SwapOutcome { |
| 396 | if a.Host == "h" && a.Cause != activator.CauseRetry { |
| 397 | return activator.SwapFailed |
| 398 | } |
| 399 | return activator.SwapDone |
| 400 | } |
| 401 | h.act.mu.Unlock() |
| 402 | gateH2 := h.act.hold("h") // h's next attempt parks in Swap |
| 403 | defer close(gateH2) |
| 404 | close(gateH) |
| 405 | waitFor(t, "h's attempt after the stale failure", func() bool { |
| 406 | calls, started, _ := h.act.snapshot() |
| 407 | n := 0 |
| 408 | for _, st := range started { |
| 409 | if st == "h:v2" { |
| 410 | n++ |
| 411 | } |
| 412 | } |
| 413 | return len(calls) >= 1 && n == 2 |
| 414 | }) |
| 415 | } else { |
| 416 | defer close(gateH) |
| 417 | h.m.OnGen(activator.GenEvent{Kind: activator.GenFailed, Route: h.rows.get("w"), Failed: plan.Boot, Mode: plan.Mode, |
| 418 | FirstWake: true, Reason: "health check failed", Token: plan.Token, ConfigRev: plan.ConfigRev}) |
| 419 | } |
| 420 | defer close(gateY) |
| 421 | |
| 422 | s.mu.Lock() |
| 423 | halted := s.q.cohortFor(cohortKey("v2", classAdopt)).halted |
| 424 | y := s.q.hosts["y"] |
| 425 | yRunning, yRev, yNewest := y.running != nil, uint64(0), y.buildRev |
| 426 | if y.running != nil { |
| 427 | yRev = y.running.SchedRev |
| 428 | } |
| 429 | s.mu.Unlock() |
| 430 | if halted { |
| 431 | t.Fatal("a stale outcome halted the cohort") |
| 432 | } |
| 433 | if !yRunning || yRev != revY || yNewest != revY { |
| 434 | t.Fatalf("y's attempt: running %v, rev %d, newest %d; want it running unsuperseded at %d", yRunning, yRev, yNewest, revY) |
| 435 | } |
| 436 | if h.book.Blocks("bab", "blog", "v2", host, false) || h.book.HeldDown("bab", "blog", "v2", host) { |
| 437 | t.Fatalf("the stale outcome took %s's retry back", host) |
| 438 | } |
| 439 | if retry == "pointer" { |
| 440 | if _, held := h.book.Lookup("bab", "blog", "v2"); held { |
| 441 | t.Fatal("the stale outcome recreated the cleared record") |
| 442 | } |
| 443 | } |
| 444 | if kind == "attempt" { |
| 445 | calls, _, _ := h.act.snapshot() |
| 446 | if c := calls[0].a; c.Host != "h" || calls[0].outcome != activator.SwapFailed { |
| 447 | t.Fatalf("first finished call %+v: want h's stale failure", calls[0]) |
| 448 | } |
| 449 | return |
| 450 | } |
| 451 | h.wakeHost("w") |
| 452 | if p := h.m.Plan(ctx, h.rows.get("w")); p.Refuse != "" || p.Boot.Version != "v2" || !p.FirstWake { |
| 453 | t.Fatalf("w's next plan %+v: want the retried v2", p) |
| 454 | } |
| 455 | } |
| 456 | |
amadan