| 1 | package rollout |
| 2 | |
| 3 | import ( |
| 4 | "testing" |
| 5 | "time" |
| 6 | |
| 7 | "github.com/carlosframework/platform/internal/activator" |
| 8 | ) |
| 9 | |
| 10 | // currentView is every slot as the activator would report it. |
| 11 | func currentView(q *appQueue) map[string]string { |
| 12 | v := map[string]string{} |
| 13 | for host, h := range q.hosts { |
| 14 | v[host] = h.serving |
| 15 | } |
| 16 | return v |
| 17 | } |
| 18 | |
| 19 | func releaseAll(q *appQueue, build string, hosts ...string) { |
| 20 | for _, h := range hosts { |
| 21 | q.wantBuild(h, activator.CauseRelease, build, t0) |
| 22 | } |
| 23 | } |
| 24 | |
| 25 | func TestTheCanaryRunsAloneUntilItSettles(t *testing.T) { |
| 26 | q := awakeQueue(noBlocks, "a", "b", "c") |
| 27 | q.concurrency = 2 |
| 28 | releaseAll(q, "v2", "a", "b", "c") |
| 29 | first := q.admit(currentView(q), t0) |
| 30 | if len(first) != 1 || !first[0].Canary { |
| 31 | t.Fatalf("admitted %+v, want exactly one canary", first) |
| 32 | } |
| 33 | if more := q.admit(currentView(q), t0); len(more) != 0 { |
| 34 | t.Fatalf("admitted %+v while the canary has not settled", more) |
| 35 | } |
| 36 | q.finish(first[0], activator.SwapDone) |
| 37 | rest := q.admit(currentView(q), t0) |
| 38 | if len(rest) != 2 || rest[0].Canary || rest[1].Canary { |
| 39 | t.Fatalf("after the settle admitted %+v, want the other two and no canary", rest) |
| 40 | } |
| 41 | } |
| 42 | |
| 43 | func TestSwapConcurrencyIsNeverExceeded(t *testing.T) { |
| 44 | q := awakeQueue(noBlocks, "a", "b", "c", "d") |
| 45 | q.concurrency = 1 |
| 46 | releaseAll(q, "v2", "a", "b", "c", "d") |
| 47 | // Four attempts need four rounds. The cap turns a queue that stops |
| 48 | // admitting into a failure instead of a test that spins until its |
| 49 | // timeout on a shared box. |
| 50 | const maxRounds = 100 |
| 51 | for round, done := 0, 0; done < 4; round++ { |
| 52 | if round == maxRounds { |
| 53 | t.Fatalf("%d of 4 attempts finished after %d admission rounds: the queue stopped admitting", done, maxRounds) |
| 54 | } |
| 55 | ds := q.admit(currentView(q), t0) |
| 56 | if q.running > q.concurrency || len(ds) > 1 { |
| 57 | t.Fatalf("running %d, admitted %d, with concurrency %d", q.running, len(ds), q.concurrency) |
| 58 | } |
| 59 | for _, d := range ds { |
| 60 | q.finish(d, activator.SwapDone) |
| 61 | done++ |
| 62 | } |
| 63 | } |
| 64 | } |
| 65 | |
| 66 | func TestAnAbandonedCanaryIsReplaced(t *testing.T) { |
| 67 | q := awakeQueue(noBlocks, "a", "b") |
| 68 | releaseAll(q, "v2", "a", "b") |
| 69 | first := q.admit(currentView(q), t0) |
| 70 | q.finish(first[0], activator.SwapAbandoned) |
| 71 | next := q.admit(currentView(q), t0) |
| 72 | if len(next) != 1 || !next[0].Canary { |
| 73 | t.Fatalf("after an abandoned canary admitted %+v, want a new canary", next) |
| 74 | } |
| 75 | } |
| 76 | |
| 77 | func TestHaltOnTheFirstFailureSparesInstancesServingTheBuild(t *testing.T) { |
| 78 | b := &FailureBook{} |
| 79 | q := newAppQueue("bab", "blog", func(build, host string, served bool) bool { |
| 80 | return b.Blocks("bab", "blog", build, host, served) |
| 81 | }) |
| 82 | for _, h := range []string{"a", "b", "c"} { |
| 83 | q.observe(h, "v1", "v1", t0) |
| 84 | } |
| 85 | q.observe("d", "v2", "v2", t0) // d served v2 before it failed elsewhere |
| 86 | q.concurrency = 3 |
| 87 | releaseAll(q, "v2", "a", "b", "c", "d") |
| 88 | canary := q.admit(currentView(q), t0) |
| 89 | if len(canary) != 1 { |
| 90 | t.Fatalf("admitted %+v", canary) |
| 91 | } |
| 92 | if !q.finish(canary[0], activator.SwapFailed) { |
| 93 | t.Fatal("a failed release attempt goes through the cohort-failure transition") |
| 94 | } |
| 95 | if _, _, err := failCohort(b, q, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: canary[0].ID.Host, At: t0}); err != nil { |
| 96 | t.Fatal(err) |
| 97 | } |
| 98 | q.wantRestart("d", "r1", t0) |
| 99 | ds := q.admit(currentView(q), t0) |
| 100 | for _, d := range ds { |
| 101 | if d.ID.Build == "v2" && d.Class == classAdopt { |
| 102 | t.Fatalf("admitted %+v after the halt", d.ID) |
| 103 | } |
| 104 | } |
| 105 | if len(ds) != 1 || ds[0].ID.Host != "d" || ds[0].ID.Cause != activator.CauseRestart { |
| 106 | t.Fatalf("admitted %+v, want only d's restart onto the v2 it serves (spec §8.4)", ds) |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | // orderBook watches the transition's order: when the record is written, the |
| 111 | // attempts past spawn must not yet have been superseded. |
| 112 | type orderBook struct { |
| 113 | FailureBook |
| 114 | q *appQueue |
| 115 | host string |
| 116 | atRecord uint64 |
| 117 | } |
| 118 | |
| 119 | func (o *orderBook) Record(r FailureRecord) (RecordResult, error) { |
| 120 | o.atRecord = o.q.newest(o.host) |
| 121 | return o.FailureBook.Record(r) |
| 122 | } |
| 123 | |
| 124 | func TestTheRecordComesBeforeTheSupersession(t *testing.T) { |
| 125 | q := awakeQueue(noBlocks, "a", "b") |
| 126 | q.concurrency = 2 |
| 127 | releaseAll(q, "v2", "a", "b") |
| 128 | canary := q.admit(currentView(q), t0) |
| 129 | q.finish(canary[0], activator.SwapDone) |
| 130 | running := q.admit(currentView(q), t0) // b, now past spawn |
| 131 | if len(running) != 1 { |
| 132 | t.Fatalf("admitted %+v", running) |
| 133 | } |
| 134 | before := q.newest(running[0].ID.Host) |
| 135 | ob := &orderBook{q: q, host: running[0].ID.Host} |
| 136 | _, superseded, _ := failCohort(ob, q, FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "c", At: t0}) |
| 137 | if ob.atRecord != before { |
| 138 | t.Fatal("attempts were superseded before the failure record was written (spec §8.1 order)") |
| 139 | } |
| 140 | if len(superseded) != 1 || q.newest(running[0].ID.Host) == before { |
| 141 | t.Fatalf("superseded %v: an attempt past spawn onto the failed build must be abandoned at its re-check", superseded) |
| 142 | } |
| 143 | } |
| 144 | |
| 145 | func TestAWakingInstanceKeepsItsPlaceAndASleeperLosesItsRestart(t *testing.T) { |
| 146 | q := awakeQueue(noBlocks, "a", "b") |
| 147 | q.wantRestart("a", "r1", t0) |
| 148 | q.wantRestart("b", "r1", t0) |
| 149 | v := currentView(q) |
| 150 | delete(v, "a") // waking |
| 151 | v["b"] = "" // asleep |
| 152 | if ds := q.admit(v, t0); len(ds) != 0 { |
| 153 | t.Fatalf("admitted %+v", ds) |
| 154 | } |
| 155 | if q.hosts["a"].wantRestart != "r1" { |
| 156 | t.Fatal("a waking instance keeps its place (spec §6.2)") |
| 157 | } |
| 158 | if q.hosts["b"].wantRestart != "" { |
| 159 | t.Fatal("a restart never wakes a sleeper") |
| 160 | } |
| 161 | } |
| 162 | |
| 163 | func TestCohortsAreKeyedByDesiredBuild(t *testing.T) { |
| 164 | q := awakeQueue(noBlocks, "a", "b") |
| 165 | q.concurrency = 2 |
| 166 | q.wantBuild("a", activator.CauseRelease, "v2", t0) // one channel |
| 167 | q.wantBuild("b", activator.CauseRelease, "v3", t0) // another |
| 168 | ds := q.admit(currentView(q), t0) |
| 169 | if len(ds) != 2 || !ds[0].Canary || !ds[1].Canary { |
| 170 | t.Fatalf("admitted %+v, want each cohort's own canary", ds) |
| 171 | } |
| 172 | } |
| 173 | |
| 174 | func TestARetriedInstanceRunsAsACanaryThroughAHalt(t *testing.T) { |
| 175 | q := awakeQueue(noBlocks, "a", "b") |
| 176 | releaseAll(q, "v2", "a", "b") |
| 177 | q.halt("v2", "health check failed") |
| 178 | if ds := q.admit(currentView(q), t0); len(ds) != 0 { |
| 179 | t.Fatalf("admitted %+v from a halted cohort", ds) |
| 180 | } |
| 181 | q.retryHost("b", "v2") |
| 182 | ds := q.admit(currentView(q), t0) |
| 183 | if len(ds) != 1 || ds[0].ID.Host != "b" || !ds[0].Canary { |
| 184 | t.Fatalf("admitted %+v, want b alone, as a canary", ds) |
| 185 | } |
| 186 | q.unhalt("v2") |
| 187 | q.finish(ds[0], activator.SwapDone) |
| 188 | if ds := q.admit(currentView(q), t0); len(ds) != 1 || ds[0].ID.Host != "a" { |
| 189 | t.Fatalf("after the app retry admitted %+v", ds) |
| 190 | } |
| 191 | } |
| 192 | |
| 193 | func TestAFailedConfigAttemptWaitsOutItsBackoff(t *testing.T) { |
| 194 | q := awakeQueue(noBlocks, "a") |
| 195 | q.wantConfig("a", "c1", t0) |
| 196 | q.configFailed("a", t0.Add(time.Minute)) |
| 197 | if ds := q.admit(currentView(q), t0); len(ds) != 0 { |
| 198 | t.Fatal("admitted a config attempt inside its backoff") |
| 199 | } |
| 200 | if got := q.nextDue(t0); !got.Equal(t0.Add(time.Minute)) { |
| 201 | t.Fatalf("nextDue = %v, want the backoff's end", got) |
| 202 | } |
| 203 | if ds := q.admit(currentView(q), t0.Add(time.Minute)); len(ds) != 1 { |
| 204 | t.Fatal("the config attempt did not come back after its backoff") |
| 205 | } |
| 206 | } |
| 207 | |
| 208 | // Task 5's pending() does not read notBefore, so admission must: a staging |
| 209 | // refusal's wait holds every attempt for the host, not only config ones. |
| 210 | func TestAStagingRefusalWaitsOutItsDeferral(t *testing.T) { |
| 211 | q := awakeQueue(noBlocks, "a", "b") |
| 212 | q.concurrency = 2 |
| 213 | releaseAll(q, "v2", "a", "b") |
| 214 | q.deferHost("a", t0.Add(30*time.Second)) |
| 215 | ds := q.admit(currentView(q), t0) |
| 216 | if len(ds) != 1 || ds[0].ID.Host != "b" { |
| 217 | t.Fatalf("admitted %+v, want only b: a is deferred", ds) |
| 218 | } |
| 219 | if got := q.nextDue(t0); !got.Equal(t0.Add(30 * time.Second)) { |
| 220 | t.Fatalf("nextDue = %v, want the deferral's end", got) |
| 221 | } |
| 222 | q.finish(ds[0], activator.SwapDone) |
| 223 | if ds := q.admit(currentView(q), t0.Add(29*time.Second)); len(ds) != 0 { |
| 224 | t.Fatalf("admitted %+v inside the deferral", ds) |
| 225 | } |
| 226 | if ds := q.admit(currentView(q), t0.Add(30*time.Second)); len(ds) != 1 || ds[0].ID.Host != "a" { |
| 227 | t.Fatalf("after the deferral admitted %+v, want a", ds) |
| 228 | } |
| 229 | if got := q.nextDue(t0.Add(30 * time.Second)); !got.IsZero() { |
| 230 | t.Fatalf("nextDue = %v, want zero once nothing waits", got) |
| 231 | } |
| 232 | } |
| 233 | |
| 234 | // A deferred host never holds the cohort's canary slot: the next host in the |
| 235 | // queue takes it, so one refusal does not stall the whole rollout. |
| 236 | func TestADeferredHostDoesNotHoldTheCanarySlot(t *testing.T) { |
| 237 | q := awakeQueue(noBlocks, "a", "b") |
| 238 | releaseAll(q, "v2", "a", "b") |
| 239 | q.deferHost("a", t0.Add(time.Minute)) |
| 240 | ds := q.admit(currentView(q), t0) |
| 241 | if len(ds) != 1 || ds[0].ID.Host != "b" || !ds[0].Canary { |
| 242 | t.Fatalf("admitted %+v, want b as the canary", ds) |
| 243 | } |
| 244 | } |
| 245 | |
| 246 | // A forced canary overrides the halt and nothing else: spec §6.2 says no other |
| 247 | // swap in a cohort starts until its first one settles, and swap_concurrency |
| 248 | // counts the canary too. So `deploy --retry` while the retried host's swap |
| 249 | // still runs must not let the next host become a second canary beside it. |
| 250 | func TestARetryThenAnUnhaltLeavesOneCanaryUntilItSettles(t *testing.T) { |
| 251 | q := awakeQueue(noBlocks, "a", "b", "c") |
| 252 | q.concurrency = 2 |
| 253 | releaseAll(q, "v2", "a", "b", "c") |
| 254 | q.halt("v2", "health check failed") |
| 255 | q.retryHost("b", "v2") |
| 256 | forced := q.admit(currentView(q), t0) |
| 257 | if len(forced) != 1 || forced[0].ID.Host != "b" || !forced[0].Canary { |
| 258 | t.Fatalf("admitted %+v, want b alone, as a canary", forced) |
| 259 | } |
| 260 | q.unhalt("v2") |
| 261 | if more := q.admit(currentView(q), t0); len(more) != 0 { |
| 262 | t.Fatalf("admitted %+v beside the retried canary, which has not settled", more) |
| 263 | } |
| 264 | q.finish(forced[0], activator.SwapDone) |
| 265 | rest := q.admit(currentView(q), t0) |
| 266 | if len(rest) != 2 || rest[0].Canary || rest[1].Canary { |
| 267 | t.Fatalf("after the settle admitted %+v, want a and c and no canary", rest) |
| 268 | } |
| 269 | } |
| 270 | |
| 271 | // Retrying a host in a cohort that is not halted gives it no way round an |
| 272 | // unsettled canary. |
| 273 | func TestARetriedInstanceWaitsForTheCohortsUnsettledCanary(t *testing.T) { |
| 274 | q := awakeQueue(noBlocks, "a", "b") |
| 275 | q.concurrency = 2 |
| 276 | releaseAll(q, "v2", "a", "b") |
| 277 | first := q.admit(currentView(q), t0) |
| 278 | if len(first) != 1 || first[0].ID.Host != "a" || !first[0].Canary { |
| 279 | t.Fatalf("admitted %+v, want a as the canary", first) |
| 280 | } |
| 281 | q.retryHost("b", "v2") |
| 282 | if more := q.admit(currentView(q), t0); len(more) != 0 { |
| 283 | t.Fatalf("admitted %+v beside a's unsettled canary", more) |
| 284 | } |
| 285 | q.finish(first[0], activator.SwapDone) |
| 286 | if ds := q.admit(currentView(q), t0); len(ds) != 1 || ds[0].ID.Host != "b" || !ds[0].Canary { |
| 287 | t.Fatalf("after the settle admitted %+v, want b as a canary", ds) |
| 288 | } |
| 289 | } |
| 290 | |
| 291 | // Spec §8.4: the next attempt after `instances retry` runs as a canary. That |
| 292 | // holds in a cohort that is not halted (a halt lives in memory and an agent |
| 293 | // restart loses it, and a second retry can follow a settled canary), where |
| 294 | // the retried host would otherwise run as an ordinary swap with no settle |
| 295 | // window. |
| 296 | func TestARetriedInstanceIsACanaryInACohortThatIsNotHalted(t *testing.T) { |
| 297 | q := awakeQueue(noBlocks, "a", "b") |
| 298 | releaseAll(q, "v2", "a", "b") |
| 299 | first := q.admit(currentView(q), t0) |
| 300 | if len(first) != 1 || first[0].ID.Host != "a" || !first[0].Canary { |
| 301 | t.Fatalf("admitted %+v, want a as the canary", first) |
| 302 | } |
| 303 | q.finish(first[0], activator.SwapDone) |
| 304 | q.retryHost("b", "v2") |
| 305 | ds := q.admit(currentView(q), t0) |
| 306 | if len(ds) != 1 || ds[0].ID.Host != "b" || !ds[0].Canary { |
| 307 | t.Fatalf("admitted %+v, want b as a canary", ds) |
| 308 | } |
| 309 | // It claims the slot: the cohort has an unsettled canary again. |
| 310 | if c := q.cohorts[ds[0].Cohort]; c.canary != "b" || c.settled { |
| 311 | t.Fatalf("cohort %+v, want b's canary unsettled", *c) |
| 312 | } |
| 313 | } |
| 314 | |
| 315 | // A halted cohort whose canary is still running (halt supersedes it, but it |
| 316 | // has not been abandoned yet) holds a retried host back too. |
| 317 | func TestARetriedInstanceWaitsForARunningCanaryInAHaltedCohort(t *testing.T) { |
| 318 | q := awakeQueue(noBlocks, "a", "b") |
| 319 | q.concurrency = 2 |
| 320 | releaseAll(q, "v2", "a", "b") |
| 321 | first := q.admit(currentView(q), t0) |
| 322 | q.halt("v2", "another host failed") |
| 323 | q.retryHost("b", "v2") |
| 324 | if more := q.admit(currentView(q), t0); len(more) != 0 { |
| 325 | t.Fatalf("admitted %+v beside a running canary", more) |
| 326 | } |
| 327 | q.finish(first[0], activator.SwapAbandoned) |
| 328 | if ds := q.admit(currentView(q), t0); len(ds) != 1 || ds[0].ID.Host != "b" || !ds[0].Canary { |
| 329 | t.Fatalf("after the abandon admitted %+v, want b as a canary", ds) |
| 330 | } |
| 331 | } |
| 332 | |
| 333 | // Final review F8: a forced attempt that ran nothing gets its allowance |
| 334 | // back, bound to the build it was for. Since final review round 1, finding |
| 335 | // 5, that holds when the host's desire has moved on to another build too: |
| 336 | // the allowance waits for the adoption onto its build, and never lets an |
| 337 | // adoption onto the other build through. An attempt that was never forced |
| 338 | // gets nothing. |
| 339 | func TestARearmedRetryStaysBoundToItsBuild(t *testing.T) { |
| 340 | q := awakeQueue(noBlocks, "a", "b") |
| 341 | releaseAll(q, "v2", "a", "b") |
| 342 | q.halt("v2", "health check failed") |
| 343 | q.retryHost("b", "v2") |
| 344 | forced := q.admit(currentView(q), t0) |
| 345 | if len(forced) != 1 || forced[0].ID.Host != "b" || !forced[0].Forced { |
| 346 | t.Fatalf("admitted %+v, want b's forced canary", forced) |
| 347 | } |
| 348 | q.finish(forced[0], activator.SwapAbandoned) |
| 349 | q.rearmRetry(forced[0]) |
| 350 | again := q.admit(currentView(q), t0) |
| 351 | if len(again) != 1 || again[0].ID.Host != "b" || !again[0].Canary || !again[0].Forced { |
| 352 | t.Fatalf("after the re-arm admitted %+v, want b's forced canary again", again) |
| 353 | } |
| 354 | q.finish(again[0], activator.SwapAbandoned) |
| 355 | q.wantBuild("b", activator.CauseRelease, "v3", t0) // a newer build replaced the desire |
| 356 | q.rearmRetry(again[0]) |
| 357 | if got := q.hosts["b"].forceBuilds; len(got) != 1 || !got["v2"] { |
| 358 | t.Fatalf("forceBuilds = %v: want the allowance for v2 back, waiting for v2", got) |
| 359 | } |
| 360 | q.halt("v3", "health check failed") |
| 361 | if ds := q.admit(currentView(q), t0); len(ds) != 0 { |
| 362 | t.Fatalf("admitted %+v: an allowance for v2 let an adoption onto v3 through v3's halt", ds) |
| 363 | } |
| 364 | q.rearmRetry(dispatch{ID: Identity{Host: "a", Build: "v2"}}) // never forced |
| 365 | if got := q.hosts["a"].forceBuilds; len(got) != 0 { |
| 366 | t.Fatalf("forceBuilds = %v: an attempt that was never forced armed a retry", got) |
| 367 | } |
| 368 | } |
| 369 | |
| 370 | // A retry of another build must not leave a flag that lets an unrelated |
| 371 | // later attempt skip the settle, or a halt (the entitlement is its build's, |
| 372 | // final review round 1, finding 5). |
| 373 | func TestAStaleRetryDoesNotBypassTheSettleOrAHalt(t *testing.T) { |
| 374 | q := awakeQueue(noBlocks, "a", "b", "c") |
| 375 | q.concurrency = 3 |
| 376 | q.retryHost("c", "v9") // a retry of a build c does not want: nothing for v2 |
| 377 | releaseAll(q, "v2", "a", "b", "c") |
| 378 | first := q.admit(currentView(q), t0) |
| 379 | if len(first) != 1 || first[0].ID.Host != "a" { |
| 380 | t.Fatalf("admitted %+v, want only a's canary: c's old retry bought nothing", first) |
| 381 | } |
| 382 | q.halt("v2", "health check failed") |
| 383 | q.finish(first[0], activator.SwapFailed) |
| 384 | if ds := q.admit(currentView(q), t0); len(ds) != 0 { |
| 385 | t.Fatalf("admitted %+v from a halted cohort on a stale retry", ds) |
| 386 | } |
| 387 | } |
| 388 | |
amadan