| 1 | package rollout |
| 2 | |
| 3 | import ( |
| 4 | "sort" |
| 5 | "time" |
| 6 | |
| 7 | "github.com/carlosframework/platform/internal/activator" |
| 8 | ) |
| 9 | |
| 10 | // Identity is an attempt's identity (spec §6.3) without the scheduler |
| 11 | // revision, which travels beside it in dispatch. Two submissions with equal |
| 12 | // identities are one attempt. |
| 13 | type Identity struct { |
| 14 | Host string |
| 15 | Cause activator.Cause |
| 16 | Build string |
| 17 | ConfigRev string |
| 18 | RestartID string |
| 19 | } |
| 20 | |
| 21 | // attemptClass separates the two failure policies of spec §8.1. |
| 22 | type attemptClass int |
| 23 | |
| 24 | const ( |
| 25 | // classAdopt is a release, install or retry onto a build this instance |
| 26 | // has not served: its failure writes a record and halts the cohort. |
| 27 | classAdopt attemptClass = iota |
| 28 | // classRecover is a restart or config attempt, or a move back onto a |
| 29 | // build the instance has served: its failure writes no record. |
| 30 | classRecover |
| 31 | ) |
| 32 | |
| 33 | // dispatch is one attempt the queue has handed out to run. |
| 34 | type dispatch struct { |
| 35 | ID Identity |
| 36 | SchedRev uint64 |
| 37 | Canary bool |
| 38 | Class attemptClass |
| 39 | Cohort string |
| 40 | // FailedUnder is the PromotedAt of the pointer the attempt's build was |
| 41 | // promoted under, read by the scheduler (Task 8) before the attempt |
| 42 | // runs, so a failure is published with no blocking read in between |
| 43 | // (spec §8.1, §8.4). Zero: the failure's own time stands in. |
| 44 | FailedUnder time.Time |
| 45 | // Forced: the attempt is the adoption a `carlos instances retry` |
| 46 | // allowance (hostSlot.forceBuilds) let through, spent when it started. |
| 47 | // An attempt that ends with nothing of its build run gets it back |
| 48 | // (rearmRetry; final review F8). |
| 49 | Forced bool |
| 50 | // Started is when execute handed the attempt to the activator (Swap or |
| 51 | // ReplaceWorker): its execution's start, the At of a hold-ran marker it |
| 52 | // leaves (ruling 205, Task 3 fix round 1). Zero until then. |
| 53 | Started time.Time |
| 54 | // BookSeq is the failure book's snapshot (FailureBook.Snapshot), taken |
| 55 | // with Started: the order of this execution's start against the book's |
| 56 | // markers and retry clears. Its good write clears only a marker set |
| 57 | // before it (Astra R3 N6), and its failure is stale after a retry since |
| 58 | // (ruling 191). Zero until then. |
| 59 | BookSeq uint64 |
| 60 | // ExecID is this execution's identity, "a<incarnation>-<n>" from a |
| 61 | // Manager-wide counter (Manager.nextExecID), set with Started. The |
| 62 | // incarnation keeps it unique across agent restarts (final review round |
| 63 | // 1, finding 1). A hold-ran marker it leaves |
| 64 | // carries it (HoldRan.Exec). SchedRev cannot serve: an abandoned |
| 65 | // attempt is derived again with the same revision. |
| 66 | ExecID string |
| 67 | } |
| 68 | |
| 69 | // hostSlot is one instance's desired state. The attempt to run is derived |
| 70 | // from it (pending), never stored, so supersession and coalescing are the |
| 71 | // same rule: the newest desire wins. |
| 72 | type hostSlot struct { |
| 73 | host string |
| 74 | since time.Time // its place in the queue: when its current desire appeared |
| 75 | serving string // the current generation's build; "" when not awake |
| 76 | good string // GoodVersion |
| 77 | |
| 78 | wantBuild string |
| 79 | wantCause activator.Cause |
| 80 | wantRestart string |
| 81 | wantConfig string |
| 82 | configAfter time.Time // a failed config attempt's backoff (spec §8.1) |
| 83 | notBefore time.Time // a staging refusal's wait (spec §4.2) |
| 84 | |
| 85 | running *dispatch |
| 86 | buildRev uint64 // newest(host): moved only by a change of desired build |
| 87 | // forceBuilds are `carlos instances retry`'s canary entitlements, one per |
| 88 | // retried build: each lets the adoption onto its build through a halt, |
| 89 | // as a canary. Each is its build's, whatever the slot wants when the |
| 90 | // retry lands, and is spent only by that adoption, never by an attempt |
| 91 | // onto another build or a restart or config attempt (final review round |
| 92 | // 1, finding 5). They are kept per build because a retry of one build |
| 93 | // must not take away another's (final review round 2, finding 5, |
| 94 | // Astra): the book lists the host as retried for the first build, so a |
| 95 | // second retry of it clears nothing and could not give the entitlement |
| 96 | // back. dropRetries removes the ones the book no longer lists, so the |
| 97 | // set holds at most one entry per record that lists the host. |
| 98 | forceBuilds map[string]bool |
| 99 | } |
| 100 | |
| 101 | // cohort is one app's attempts on this box onto one build, of one class |
| 102 | // (spec §3). |
| 103 | type cohort struct { |
| 104 | canary string // the host of the cohort's canary, "" until one starts |
| 105 | settled bool // the canary passed its health check and its settle |
| 106 | halted bool |
| 107 | reason string |
| 108 | } |
| 109 | |
| 110 | // appQueue is one app's attempts on this box. It is not safe for concurrent |
| 111 | // use: the scheduler holds its lock around every call. |
| 112 | type appQueue struct { |
| 113 | acct, app string |
| 114 | concurrency int |
| 115 | hosts map[string]*hostSlot |
| 116 | cohorts map[string]*cohort |
| 117 | running int |
| 118 | rev uint64 |
| 119 | // blocks is the failure book's answer for this app (spec §8.4). |
| 120 | blocks func(build, host string, served bool) bool |
| 121 | } |
| 122 | |
| 123 | func newAppQueue(acct, app string, blocks func(build, host string, served bool) bool) *appQueue { |
| 124 | return &appQueue{acct: acct, app: app, concurrency: 1, hosts: map[string]*hostSlot{}, cohorts: map[string]*cohort{}, blocks: blocks} |
| 125 | } |
| 126 | |
| 127 | func (q *appQueue) nextRev() uint64 { |
| 128 | q.rev++ |
| 129 | return q.rev |
| 130 | } |
| 131 | |
| 132 | func (q *appQueue) slot(host string, now time.Time) *hostSlot { |
| 133 | h := q.hosts[host] |
| 134 | if h == nil { |
| 135 | h = &hostSlot{host: host, since: now} |
| 136 | q.hosts[host] = h |
| 137 | } |
| 138 | return h |
| 139 | } |
| 140 | |
| 141 | // newest is the revision an attempt past spawn must still carry when it |
| 142 | // switches (spec §6.3). Only a change of desired build moves it: a restart |
| 143 | // or config request that arrives meanwhile queues behind the attempt instead |
| 144 | // of abandoning it. |
| 145 | func (q *appQueue) newest(host string) uint64 { |
| 146 | if h := q.hosts[host]; h != nil { |
| 147 | return h.buildRev |
| 148 | } |
| 149 | return 0 |
| 150 | } |
| 151 | |
| 152 | // observe records the agent pass's view of one host. A running attempt owns |
| 153 | // the serving build until it finishes. |
| 154 | func (q *appQueue) observe(host, serving, good string, now time.Time) { |
| 155 | h := q.slot(host, now) |
| 156 | if h.running == nil { |
| 157 | h.serving = serving |
| 158 | } |
| 159 | h.good = good |
| 160 | } |
| 161 | |
| 162 | func (q *appQueue) touch(h *hostSlot, now time.Time) { |
| 163 | if _, queued := q.pending(h); !queued && h.running == nil { |
| 164 | h.since = now |
| 165 | } |
| 166 | } |
| 167 | |
| 168 | // wantBuild sets host's desired build: its installed build (spec §3). cause |
| 169 | // is release, install or retry. |
| 170 | func (q *appQueue) wantBuild(host string, cause activator.Cause, build string, now time.Time) bool { |
| 171 | if build == "" { |
| 172 | return false |
| 173 | } |
| 174 | h := q.slot(host, now) |
| 175 | if h.wantBuild == build { |
| 176 | if cause == activator.CauseRetry && h.wantCause != activator.CauseRetry { |
| 177 | h.wantCause = cause |
| 178 | return true |
| 179 | } |
| 180 | return false |
| 181 | } |
| 182 | q.touch(h, now) |
| 183 | h.wantBuild, h.wantCause = build, cause |
| 184 | h.buildRev = q.nextRev() // rolloutsim:M4 |
| 185 | return true |
| 186 | } |
| 187 | |
| 188 | // wantRestart records a `carlos restart` request; the newest id covers the |
| 189 | // older ones (spec §6.3). |
| 190 | func (q *appQueue) wantRestart(host, id string, now time.Time) bool { |
| 191 | h := q.slot(host, now) |
| 192 | if id == "" || h.wantRestart == id { |
| 193 | return false |
| 194 | } |
| 195 | q.touch(h, now) |
| 196 | h.wantRestart = id |
| 197 | return true |
| 198 | } |
| 199 | |
| 200 | // wantConfig records a delivered config revision that is not yet active. A |
| 201 | // newer revision replaces the pending attempt and its backoff (spec §8.1). |
| 202 | func (q *appQueue) wantConfig(host, rev string, now time.Time) bool { |
| 203 | h := q.slot(host, now) |
| 204 | if h.wantConfig == rev { |
| 205 | return false |
| 206 | } |
| 207 | q.touch(h, now) |
| 208 | h.wantConfig, h.configAfter = rev, time.Time{} |
| 209 | return true |
| 210 | } |
| 211 | |
| 212 | // configActive clears the pending config attempt once rev is active. |
| 213 | func (q *appQueue) configActive(host, rev string) { |
| 214 | if h := q.hosts[host]; h != nil && h.wantConfig == rev { |
| 215 | h.wantConfig = "" |
| 216 | } |
| 217 | } |
| 218 | |
| 219 | // dropConfig clears whatever config attempt is pending for host, whichever |
| 220 | // revision it names. It is for a host with no process: its next wake starts |
| 221 | // with the delivered file. It never creates a slot. |
| 222 | func (q *appQueue) dropConfig(host string) { |
| 223 | if h := q.hosts[host]; h != nil { |
| 224 | h.wantConfig, h.configAfter = "", time.Time{} |
| 225 | } |
| 226 | } |
| 227 | |
| 228 | // configFailed holds a config-only attempt back until its backoff ends. |
| 229 | func (q *appQueue) configFailed(host string, until time.Time) { |
| 230 | if h := q.hosts[host]; h != nil { |
| 231 | h.configAfter = until |
| 232 | } |
| 233 | } |
| 234 | |
| 235 | // deferHost holds every attempt for host back until until: a staging refusal |
| 236 | // waits one stage-relay window for root to restage. |
| 237 | func (q *appQueue) deferHost(host string, until time.Time) { |
| 238 | if h := q.hosts[host]; h != nil { |
| 239 | h.notBefore = until |
| 240 | } |
| 241 | } |
| 242 | |
| 243 | // pending derives the attempt host should run next, if any. Only an awake |
| 244 | // instance with nothing running has one. A desired build the failure book |
| 245 | // holds falls through to the instance's restart or config work, which goes |
| 246 | // ahead on the build it serves (spec §8.4). |
| 247 | func (q *appQueue) pending(h *hostSlot) (dispatch, bool) { |
| 248 | if h.running != nil || h.serving == "" { |
| 249 | return dispatch{}, false |
| 250 | } |
| 251 | id := Identity{Host: h.host, ConfigRev: h.wantConfig, RestartID: h.wantRestart} |
| 252 | switch { |
| 253 | case h.wantBuild != "" && h.wantBuild != h.serving && !q.blocks(h.wantBuild, h.host, h.wantBuild == h.good): |
| 254 | id.Cause, id.Build = h.wantCause, h.wantBuild |
| 255 | case h.wantRestart != "": |
| 256 | id.Cause, id.Build = activator.CauseRestart, h.serving |
| 257 | case h.wantConfig != "": |
| 258 | id.Cause, id.Build = activator.CauseConfig, h.serving |
| 259 | default: |
| 260 | return dispatch{}, false |
| 261 | } |
| 262 | class := classRecover // rolloutsim:M2 |
| 263 | if id.Build != h.serving && id.Build != h.good { |
| 264 | class = classAdopt |
| 265 | } |
| 266 | return dispatch{ID: id, SchedRev: h.buildRev, Class: class, Cohort: cohortKey(id.Build, class)}, true |
| 267 | } |
| 268 | |
| 269 | // cohortKey names a cohort (spec §3): the attempts onto one build, split by |
| 270 | // class so a halt of the adoption cohort never stops the restarts and config |
| 271 | // changes of instances already serving the build. |
| 272 | func cohortKey(build string, class attemptClass) string { |
| 273 | if class == classAdopt { |
| 274 | return "adopt:" + build |
| 275 | } |
| 276 | return "recover:" + build |
| 277 | } |
| 278 | |
| 279 | func (q *appQueue) cohortFor(key string) *cohort { |
| 280 | c := q.cohorts[key] |
| 281 | if c == nil { |
| 282 | c = &cohort{} |
| 283 | q.cohorts[key] = c |
| 284 | } |
| 285 | return c |
| 286 | } |
| 287 | |
| 288 | // ordered is the queue order: oldest desire first, host name to break ties. |
| 289 | func (q *appQueue) ordered() []*hostSlot { |
| 290 | out := make([]*hostSlot, 0, len(q.hosts)) |
| 291 | for _, h := range q.hosts { |
| 292 | out = append(out, h) |
| 293 | } |
| 294 | sort.Slice(out, func(i, j int) bool { |
| 295 | if !out[i].since.Equal(out[j].since) { |
| 296 | return out[i].since.Before(out[j].since) |
| 297 | } |
| 298 | return out[i].host < out[j].host |
| 299 | }) |
| 300 | return out |
| 301 | } |
| 302 | |
| 303 | // admit hands out the attempts that may start now (spec §6.2). view maps each |
| 304 | // host the activator could see to the build it serves ("" when asleep); a |
| 305 | // host that is waking or stopping is absent, and keeps its place. A sleeper |
| 306 | // loses only what needs a running process: its restart request. |
| 307 | func (q *appQueue) admit(view map[string]string, now time.Time) []dispatch { |
| 308 | var out []dispatch |
| 309 | for _, h := range q.ordered() { |
| 310 | if h.running != nil { |
| 311 | continue |
| 312 | } |
| 313 | serving, seen := view[h.host] |
| 314 | if !seen { |
| 315 | continue |
| 316 | } |
| 317 | h.serving = serving |
| 318 | if serving == "" { |
| 319 | h.wantRestart = "" |
| 320 | continue |
| 321 | } |
| 322 | d, ok := q.gate(h, now) |
| 323 | if !ok { |
| 324 | continue |
| 325 | } |
| 326 | if q.running >= q.concurrency { // rolloutsim:M1 |
| 327 | break |
| 328 | } |
| 329 | q.start(h, d) |
| 330 | out = append(out, d) |
| 331 | } |
| 332 | return out |
| 333 | } |
| 334 | |
| 335 | // gate is the per-host half of admit, shared with the worker recycle hook: |
| 336 | // the host's pending attempt, if its cohort lets it run now. pending does not |
| 337 | // read the two waits, so they are enforced here: a staging refusal holds |
| 338 | // every attempt for the host until notBefore, and a failed config attempt |
| 339 | // waits out its backoff. A cohort's first attempt is its canary and runs |
| 340 | // alone until it settles, and that holds for a retried host too: a retry |
| 341 | // makes the host's next adoption a canary and lifts the halt for it, but it |
| 342 | // still waits for any other canary of the cohort that is running and |
| 343 | // unsettled. A halted adoption cohort admits nothing but such a retried host. |
| 344 | func (q *appQueue) gate(h *hostSlot, now time.Time) (dispatch, bool) { |
| 345 | d, ok := q.pending(h) |
| 346 | if !ok || now.Before(h.notBefore) { |
| 347 | return dispatch{}, false |
| 348 | } |
| 349 | if d.ID.Cause == activator.CauseConfig && now.Before(h.configAfter) { |
| 350 | return dispatch{}, false |
| 351 | } |
| 352 | c := q.cohortFor(d.Cohort) |
| 353 | // The flag is tied to the build it was set for, so a retry on a host with |
| 354 | // nothing pending cannot lift a later cohort's halt or turn an unrelated |
| 355 | // attempt into a canary. |
| 356 | forced := d.Class == classAdopt && h.forceBuilds[d.ID.Build] |
| 357 | switch { |
| 358 | case forced: |
| 359 | // The next attempt after a retry is a canary (spec §8.4) whether or |
| 360 | // not the cohort is halted: halts live in memory and a restart loses |
| 361 | // them, and a second retry can follow a settled canary. It still |
| 362 | // waits out another host's running canary. |
| 363 | if c.canary != "" && c.canary != h.host && !c.settled { |
| 364 | return dispatch{}, false |
| 365 | } |
| 366 | d.Canary, d.Forced = true, true |
| 367 | case c.halted && d.Class == classAdopt: |
| 368 | return dispatch{}, false |
| 369 | case c.canary == "": |
| 370 | d.Canary = true |
| 371 | case !c.settled: |
| 372 | return dispatch{}, false |
| 373 | } |
| 374 | return d, true |
| 375 | } |
| 376 | |
| 377 | // start marks d running. A cohort's canary is claimed here, at the moment it |
| 378 | // actually starts, so a host gated out by the concurrency limit never holds |
| 379 | // the canary slot. A retried host's canary claims it like any other, and a |
| 380 | // settled flag left over from before the halt starts again from false. |
| 381 | func (q *appQueue) start(h *hostSlot, d dispatch) { |
| 382 | if d.Canary { |
| 383 | c := q.cohortFor(d.Cohort) |
| 384 | c.canary, c.settled = h.host, false |
| 385 | } |
| 386 | if d.Forced { |
| 387 | // Only the adoption the entitlement let through spends it (final |
| 388 | // review round 1, finding 5): a restart, a config attempt or an |
| 389 | // adoption onto another build starting first leaves it for later, |
| 390 | // and the other builds' entitlements stay (round 2, finding 5). |
| 391 | delete(h.forceBuilds, d.ID.Build) |
| 392 | } |
| 393 | h.running = &d |
| 394 | q.running++ |
| 395 | } |
| 396 | |
| 397 | // finish records d's outcome and reports whether it is a build failure that |
| 398 | // must go through the cohort-failure transition (spec §8.1): a failed |
| 399 | // adoption attempt. A failed restart or config attempt is reported, never |
| 400 | // recorded. |
| 401 | func (q *appQueue) finish(d dispatch, out activator.SwapOutcome) (transition bool) { |
| 402 | h := q.hosts[d.ID.Host] |
| 403 | if h == nil || h.running == nil || h.running.SchedRev != d.SchedRev || h.running.ID != d.ID { |
| 404 | return false |
| 405 | } |
| 406 | h.running = nil |
| 407 | q.running-- |
| 408 | c := q.cohorts[d.Cohort] |
| 409 | unclaim := func() { |
| 410 | if c != nil && c.canary == h.host && !c.settled { |
| 411 | c.canary = "" |
| 412 | } |
| 413 | } |
| 414 | switch out { |
| 415 | case activator.SwapDone: |
| 416 | h.serving = d.ID.Build |
| 417 | if h.wantRestart == d.ID.RestartID { |
| 418 | h.wantRestart = "" |
| 419 | } |
| 420 | if d.ID.ConfigRev != "" && h.wantConfig == d.ID.ConfigRev { |
| 421 | h.wantConfig = "" |
| 422 | } |
| 423 | if c != nil && c.canary == h.host { |
| 424 | c.settled = true |
| 425 | } |
| 426 | case activator.SwapAbandoned: |
| 427 | unclaim() |
| 428 | case activator.SwapFailed, activator.SwapDown: |
| 429 | if out == activator.SwapDown { |
| 430 | h.serving = "" |
| 431 | } |
| 432 | if d.ID.Cause == activator.CauseRestart && h.wantRestart == d.ID.RestartID { |
| 433 | h.wantRestart = "" |
| 434 | } |
| 435 | unclaim() |
| 436 | return d.Class == classAdopt |
| 437 | } |
| 438 | return false |
| 439 | } |
| 440 | |
| 441 | // halt stops build's adoption cohort, and supersedes every adoption attempt |
| 442 | // onto build already past spawn, so its re-check abandons it rather than |
| 443 | // switching (spec §8.1 step 2). It returns the superseded hosts. |
| 444 | func (q *appQueue) halt(build, reason string) (superseded []string) { |
| 445 | c := q.cohortFor(cohortKey(build, classAdopt)) |
| 446 | c.halted, c.reason = true, reason |
| 447 | for _, h := range q.hosts { |
| 448 | if h.running != nil && h.running.Class == classAdopt && h.running.ID.Build == build { |
| 449 | h.buildRev = q.nextRev() |
| 450 | superseded = append(superseded, h.host) |
| 451 | } |
| 452 | } |
| 453 | sort.Strings(superseded) |
| 454 | return superseded |
| 455 | } |
| 456 | |
| 457 | // unhalt reopens build's adoption cohort after `carlos deploy --retry`: its |
| 458 | // next attempt is a fresh canary. A canary still running (a retried host's |
| 459 | // swap) stays the cohort's canary, so nothing else starts until it settles. |
| 460 | func (q *appQueue) unhalt(build string) { |
| 461 | if c := q.cohorts[cohortKey(build, classAdopt)]; c != nil { |
| 462 | running := "" |
| 463 | if !c.settled { |
| 464 | running = c.canary |
| 465 | } |
| 466 | *c = cohort{canary: running} |
| 467 | } |
| 468 | } |
| 469 | |
| 470 | // retryHost is `carlos instances retry <host>` naming build: the host's |
| 471 | // adoption onto build runs as a canary even through a halt, now if the slot |
| 472 | // wants build, or whenever it comes to want it (final review round 1, |
| 473 | // finding 5, Astra: a retry of a build the slot did not want yet used to |
| 474 | // leave the host exempt in the book but with no admission through the |
| 475 | // halt, and no later retry could give it one). The entitlement is bound to |
| 476 | // build: it never lets an adoption onto another build through, and a |
| 477 | // retry of another build leaves it standing (final review round 2, finding |
| 478 | // 5). |
| 479 | func (q *appQueue) retryHost(host, build string) { |
| 480 | if h := q.hosts[host]; h != nil && build != "" { |
| 481 | h.arm(build) |
| 482 | } |
| 483 | } |
| 484 | |
| 485 | // arm binds a retry entitlement for build to h. |
| 486 | func (h *hostSlot) arm(build string) { |
| 487 | if h.forceBuilds == nil { |
| 488 | h.forceBuilds = map[string]bool{} |
| 489 | } |
| 490 | h.forceBuilds[build] = true |
| 491 | } |
| 492 | |
| 493 | // dropRetries drops every entitlement whose build the failure book no |
| 494 | // longer lists the host as retried for (retried false): the record was |
| 495 | // cleared by a signed pointer or pruned, or a later failure of the host |
| 496 | // spent the retry. Such an entitlement could only make a later record of |
| 497 | // the build, one no retry has touched, admit the host through its halt |
| 498 | // (final review round 2, finding 5). The caller holds the scheduler's |
| 499 | // lock; retried may take the book's, which comes after it. |
| 500 | func (q *appQueue) dropRetries(retried func(build, host string) bool) { |
| 501 | for _, h := range q.hosts { |
| 502 | for build := range h.forceBuilds { |
| 503 | if !retried(build, h.host) { |
| 504 | delete(h.forceBuilds, build) |
| 505 | } |
| 506 | } |
| 507 | if len(h.forceBuilds) == 0 { |
| 508 | h.forceBuilds = nil |
| 509 | } |
| 510 | } |
| 511 | } |
| 512 | |
| 513 | // rearmRetry gives back the retry allowance a forced attempt spent at its |
| 514 | // start (final review F8), for an attempt that ended with nothing of its |
| 515 | // build run: abandoned before its exec, or refused as a staging error. Spec |
| 516 | // §8.4 promises the retried host's next adoption runs as a canary, and that |
| 517 | // adoption has not happened yet. The allowance stays bound to the build, |
| 518 | // and comes back even when the host now wants another build, as a retry |
| 519 | // for a build not yet wanted keeps it (finding 5). Other builds' |
| 520 | // allowances are untouched (round 2, finding 5). If the book has stopped |
| 521 | // listing the host as retried for the build meanwhile, the next pass drops |
| 522 | // it again (dropRetries). An attempt whose build ran spent the allowance |
| 523 | // for good, whatever its outcome. The caller holds the scheduler's lock, |
| 524 | // in the step that frees the attempt's slot. |
| 525 | func (q *appQueue) rearmRetry(d dispatch) { |
| 526 | h := q.hosts[d.ID.Host] |
| 527 | if !d.Forced || h == nil { |
| 528 | return |
| 529 | } |
| 530 | h.arm(d.ID.Build) |
| 531 | } |
| 532 | |
| 533 | // nextDue is the earliest moment a deferred attempt becomes admissible, so the |
| 534 | // scheduler's goroutine knows when to look again; zero when nothing waits. |
| 535 | func (q *appQueue) nextDue(now time.Time) time.Time { |
| 536 | var next time.Time |
| 537 | consider := func(t time.Time) { |
| 538 | if t.After(now) && (next.IsZero() || t.Before(next)) { |
| 539 | next = t |
| 540 | } |
| 541 | } |
| 542 | for _, h := range q.hosts { |
| 543 | consider(h.notBefore) |
| 544 | if h.wantConfig != "" { |
| 545 | consider(h.configAfter) |
| 546 | } |
| 547 | } |
| 548 | return next |
| 549 | } |
| 550 | |
| 551 | // recorder is the failure book's write half, an interface so a test can |
| 552 | // watch the transition's order. |
| 553 | type recorder interface { |
| 554 | Record(FailureRecord) (RecordResult, error) |
| 555 | } |
| 556 | |
| 557 | // failCohort is the cohort-failure transition of spec §8.1, in its order: |
| 558 | // (1) the failure record, before anything else can act on the failure; |
| 559 | // (2) queued attempts onto the build are cancelled, because the record now |
| 560 | // blocks them, and attempts past spawn are superseded; (3) wakes of the |
| 561 | // cohort boot the good build, which needs nothing here: the planner reads |
| 562 | // the record, and nothing is repointed; (4) instances that have served the |
| 563 | // build are untouched, because the record never blocks them. The caller |
| 564 | // holds the scheduler's lock for q, so the record, the halt and the freed |
| 565 | // slot (finish) are one critical section. A failed file write still returns |
| 566 | // the record held in memory, so the transition never stops halfway. |
| 567 | // |
| 568 | // A failure the book reports PreRetry began before a retry that has since |
| 569 | // cleared its build (ruling 191): it is stale, and none of the transition |
| 570 | // runs (plan review finding 3). Halting would stop the cohort the retry |
| 571 | // reopened and supersede attempts the retry let start, though the record |
| 572 | // that would justify it was never written. Every other result halts, an |
| 573 | // Unchanged one included: the record it found still holds the build, and a |
| 574 | // halt lives in memory beside it (an agent restart loses it). |
| 575 | func failCohort(book recorder, q *appQueue, rec FailureRecord) (res RecordResult, superseded []string, err error) { |
| 576 | res, err = book.Record(rec) // rolloutsim:M3 |
| 577 | if res == PreRetry { // rolloutsim:M11 |
| 578 | return res, nil, err |
| 579 | } |
| 580 | superseded = q.halt(rec.Build, rec.Reason) |
| 581 | return res, superseded, err |
| 582 | } |
| 583 | |
amadan