| 1 | package main |
| 2 | |
| 3 | // deploywatch.go is `carlos deploy`'s watch once boxes report their rollouts |
| 4 | // (zero-downtime spec §9.1): completion is tied to the build being watched, |
| 5 | // instance by instance, box by box. A host its box reports on is watched |
| 6 | // through that report and never requested over HTTP, because a request would |
| 7 | // wake an asleep instance the spec counts as done once its box has installed |
| 8 | // the build. A host no box reports on is watched as it always was |
| 9 | // (checkTarget). With no report at all, the watch IS watchVersion. |
| 10 | |
| 11 | import ( |
| 12 | "context" |
| 13 | "encoding/json" |
| 14 | "errors" |
| 15 | "fmt" |
| 16 | "io" |
| 17 | "net/http" |
| 18 | "net/url" |
| 19 | "sort" |
| 20 | "strings" |
| 21 | "sync" |
| 22 | "time" |
| 23 | |
| 24 | "github.com/carlosframework/platform/internal/rolloutwords" |
| 25 | "github.com/carlosframework/platform/internal/status" |
| 26 | ) |
| 27 | |
| 28 | const ( |
| 29 | watchIntro = "watching %s roll out box by box: an awake instance counts once the new build has taken over, an asleep one once its box has installed it" |
| 30 | watchWordSwapped = "swapped" |
| 31 | watchWordInstalled = "installed" |
| 32 | watchAsleepTail = "starts on its next request" |
| 33 | watchWordFailed = "failed" |
| 34 | watchWordHeld = "held" |
| 35 | watchHeldTail = "stayed on %s because %s failed on this box" |
| 36 | watchBox = "box %s: %s" |
| 37 | watchCountSwapped = "%d swapped" |
| 38 | watchCountAsleep = "%d asleep with it installed" |
| 39 | watchCountFailed = "%d failed" |
| 40 | watchCountHeld = "%d stayed on their last good build" |
| 41 | watchFailedSummary = "build %s did not reach every instance: %d failed and %d stayed on their last good build. Fix it and deploy a new build, or try this one again: carlos promote --app %s --retry %s %s" |
| 42 | watchHoldDownSummary = "an instance that swaps in hold mode is down, with nothing to fall back to: carlos rollback --app %s" |
| 43 | watchTimeout = "timed out waiting for build %s:" |
| 44 | watchPendingNotInstalled = "its box has not installed this build yet" |
| 45 | watchPendingOther = "its box has %s installed, not this build" |
| 46 | watchPendingQueued = "queued" |
| 47 | watchPendingRunning = "starting beside the running build" |
| 48 | watchPendingSettling = "settling: making sure it stays up for 10 seconds" |
| 49 | watchPendingWaiting = "waiting for its box to start it" |
| 50 | watchPendingDark = "its box stopped reporting" |
| 51 | watchPendingUnlisted = "the console's instance list was cut short before it, so its state is unknown" |
| 52 | watchBoxBusy = "box %s is still working through this app's instances" |
| 53 | watchListFailed = "the console's instance list could not be read, so the rollout could not be followed" |
| 54 | watchPendingUnread = "the console could not read every box's report, so its state is unknown" |
| 55 | ) |
| 56 | |
| 57 | // watchList is one answer of the console's instance list, as the watch |
| 58 | // reads it. incomplete is the console's rollout_incomplete: some box's |
| 59 | // report could not be read, so a host with no report may yet have one. |
| 60 | type watchList struct { |
| 61 | recs []apiInstanceRecord |
| 62 | truncated bool |
| 63 | incomplete bool |
| 64 | } |
| 65 | |
| 66 | // listStatusError is a list the console answered with a status other than |
| 67 | // 200. The status decides what the watch makes of it (listAbsent). |
| 68 | type listStatusError struct { |
| 69 | status int |
| 70 | err error |
| 71 | } |
| 72 | |
| 73 | func (e *listStatusError) Error() string { return e.err.Error() } |
| 74 | func (e *listStatusError) Unwrap() error { return e.err } |
| 75 | |
| 76 | // listAbsent reports whether a failed list is evidence enough to fall back |
| 77 | // to today's URL watch (Astra plan 6 R1 #4, R2 #4). A 4xx says the console |
| 78 | // has no list for this caller: a console from before the door, an app with |
| 79 | // no instances (404). A 403 counts too, on purpose: a Publish-only token can |
| 80 | // deploy (promote admits Publish) but cannot read the list (GET instances |
| 81 | // admits Instance and Deploy), so such a deploy gets exactly the watch it |
| 82 | // got before reports existed. Not a 401: the deploy has just authenticated |
| 83 | // with this token, so a 401 is a session that lapsed or a console that |
| 84 | // stumbled, and says nothing about the boxes. Not a 429 either. Those, a |
| 85 | // 5xx, a network error and an unreadable answer are retried until the |
| 86 | // deadline. |
| 87 | func listAbsent(err error) bool { |
| 88 | var se *listStatusError |
| 89 | return errors.As(err, &se) && se.status >= 400 && se.status < 500 && |
| 90 | se.status != http.StatusTooManyRequests && se.status != http.StatusUnauthorized |
| 91 | } |
| 92 | |
| 93 | // watchInstanceList is the watch's read of the instance list. It is |
| 94 | // instanceList's request, kept apart because the watch needs what that |
| 95 | // method drops: the status of a refusal, and rollout_incomplete. |
| 96 | func (s *apiSession) watchInstanceList(ctx context.Context, a apiAcct, app string) (watchList, error) { |
| 97 | status, body, err := s.getJSON(ctx, apiPath(a, app, "/instances")) |
| 98 | if err != nil { |
| 99 | return watchList{}, err |
| 100 | } |
| 101 | if status != http.StatusOK { |
| 102 | return watchList{}, &listStatusError{status: status, err: apiErrorFromBody(status, body)} |
| 103 | } |
| 104 | var res struct { |
| 105 | Instances []apiInstanceRecord `json:"instances"` |
| 106 | Truncated bool `json:"truncated"` |
| 107 | RolloutIncomplete bool `json:"rollout_incomplete"` |
| 108 | } |
| 109 | if err := json.Unmarshal(body, &res); err != nil { |
| 110 | return watchList{}, fmt.Errorf("instances list: unreadable response: %w", err) |
| 111 | } |
| 112 | return watchList{recs: res.Instances, truncated: res.Truncated, incomplete: res.RolloutIncomplete}, nil |
| 113 | } |
| 114 | |
| 115 | // deployWatchListEvery is how often the production watch reads the instance |
| 116 | // list (controller ruling R5). The list is rate limited per token (60 a |
| 117 | // minute, shared with the deploy's own calls), and each read costs the |
| 118 | // console a records list and a box join, so it keeps its own cadence; a host |
| 119 | // watched by URL is still polled every deployWatchPollEvery, as before. A |
| 120 | // failing first list is asked again on the same cadence until the deadline. |
| 121 | const ( |
| 122 | deployWatchListEvery = 5 * time.Second |
| 123 | deployWatchPollEvery = 2 * time.Second |
| 124 | ) |
| 125 | |
| 126 | // rolloutState is one targeted instance's place in the rollout of the |
| 127 | // watched build (spec §9.1): pending, or one of four final states. |
| 128 | type rolloutState string |
| 129 | |
| 130 | const ( |
| 131 | rolloutPending rolloutState = "pending" |
| 132 | rolloutSwapped rolloutState = "swapped" |
| 133 | rolloutAsleep rolloutState = "asleep" // installed, awaiting first wake |
| 134 | rolloutFailed rolloutState = "failed" |
| 135 | rolloutHeld rolloutState = "held" |
| 136 | ) |
| 137 | |
| 138 | func (s rolloutState) final() bool { |
| 139 | return s == rolloutSwapped || s == rolloutAsleep || s == rolloutFailed || s == rolloutHeld |
| 140 | } |
| 141 | |
| 142 | // attemptOpen reports whether a box's attempt word is still in progress: |
| 143 | // queued, running or settling, or a word this binary does not know (a newer |
| 144 | // box), which is safer read as unfinished than as done. Failed and down are |
| 145 | // outcomes, not attempts in progress. |
| 146 | func attemptOpen(a string) bool { |
| 147 | return a != "" && a != status.AttemptFailed && a != status.AttemptDown |
| 148 | } |
| 149 | |
| 150 | // classifyRollout decides one instance's state for build ver from its box's |
| 151 | // report. In order: |
| 152 | // - A box that cannot arm its scheduler (Unavailable) reports no per-host |
| 153 | // state, so its awake=false means nothing: pending (R6). |
| 154 | // - Until the box reports ver installed the instance is pending, however |
| 155 | // idle its scheduler. |
| 156 | // - A failure counts against ver only when the failed attempt was ver's, |
| 157 | // and a hold-mode instance left down by ver is failed too (spec §9.1 "a |
| 158 | // hold failure"). The console's join (Task 8) leaves the failed attempt |
| 159 | // only on the host the build failed on, so this is checked before |
| 160 | // held_by, which that host carries too. |
| 161 | // - Held by ver, with no failure of its own: a sibling's failure record |
| 162 | // holds it on its good build. |
| 163 | // - A transient host (its process is starting or stopping) is pending: it |
| 164 | // is neither asleep nor swapped, and its serving build says nothing |
| 165 | // settled (Astra R3 section 5 item 10). A failure or a hold it already |
| 166 | // carries still stands, as above. |
| 167 | // - An attempt in progress is pending whether or not the instance is awake |
| 168 | // (R8): a canary serving ver has not swapped until its settle closes, and |
| 169 | // an asleep instance whose first wake is under observation is not asleep. |
| 170 | // - Serving ver with nothing in progress: swapped. Asleep with nothing in |
| 171 | // progress: installed, awaiting first wake. |
| 172 | func classifyRollout(r *apiRollout, ver string) rolloutState { |
| 173 | switch { |
| 174 | case r.Unavailable != "": |
| 175 | return rolloutPending |
| 176 | case r.Installed != ver: |
| 177 | return rolloutPending |
| 178 | case r.AttemptBuild == ver && (r.Attempt == status.AttemptFailed || r.Attempt == status.AttemptDown): |
| 179 | return rolloutFailed |
| 180 | case r.HeldBy == ver: |
| 181 | return rolloutHeld |
| 182 | case r.Transient: |
| 183 | return rolloutPending |
| 184 | case attemptOpen(r.Attempt): |
| 185 | return rolloutPending |
| 186 | case r.Serving == ver: |
| 187 | return rolloutSwapped |
| 188 | case !r.Awake && r.Serving == "" && r.Attempt == "": |
| 189 | return rolloutAsleep |
| 190 | } |
| 191 | return rolloutPending |
| 192 | } |
| 193 | |
| 194 | // follows reports whether the watch follows an instance through r: a report |
| 195 | // with per-host state, or one from a box that cannot arm its scheduler for |
| 196 | // any reason but a known decline: a refusing box's instances wait, and a |
| 197 | // code this binary does not know is a newer box's problem, so neither may be |
| 198 | // woken by a poll. A box that declined overlap runs today's stop-then-start |
| 199 | // paths, so its report is no report: the instance is watched by URL, like a |
| 200 | // flag-off box's (R6; fix round 1). |
| 201 | func follows(r *apiRollout) bool { |
| 202 | return r != nil && !rolloutwords.UnavailableDeclines(r.Unavailable) |
| 203 | } |
| 204 | |
| 205 | type watchMode int |
| 206 | |
| 207 | const ( |
| 208 | modeUnknown watchMode = iota // not yet decided: a truncated list left it out, or an incomplete join had no report for it |
| 209 | modeRollout // watched through its box's report, for the rest of the watch |
| 210 | modeURL // watched by URL, as watchVersion does, until a report comes |
| 211 | ) |
| 212 | |
| 213 | type watchedHost struct { |
| 214 | t watchTarget |
| 215 | mode watchMode |
| 216 | // ro is the latest report the watch followed. It is kept when the box |
| 217 | // goes dark, so the host still counts on its box. |
| 218 | ro *apiRollout |
| 219 | // dark is set while the latest complete list carries no report for a |
| 220 | // host the watch follows (R1). It never sends the host to a URL poll. |
| 221 | dark bool |
| 222 | // unread is set while the host is undecided because the console's |
| 223 | // rollout join was incomplete, not because the list was cut short. |
| 224 | unread bool |
| 225 | // absent is set while the latest answered list does not carry the host |
| 226 | // at all: cut short, or a record the console could not read (Astra plan |
| 227 | // 6 R3). An absent host is never polled by URL, even if it was being |
| 228 | // polled before; it is cleared the next time the host is listed. |
| 229 | absent bool |
| 230 | state rolloutState |
| 231 | // fin is the report the host went final on. A final state survives |
| 232 | // missing observations, and later reports that agree the build is |
| 233 | // installed: what it printed and what it counts come from fin. A report |
| 234 | // that the box has another build installed clears it (apply). |
| 235 | fin *apiRollout |
| 236 | urlLive bool |
| 237 | urlDiag string |
| 238 | printed bool |
| 239 | } |
| 240 | |
| 241 | // urlPolling is whether this sweep requests the host's URL: it is watched by |
| 242 | // URL, is not yet live, and the console still lists it. |
| 243 | func (h *watchedHost) urlPolling() bool { |
| 244 | return h.mode == modeURL && !h.urlLive && !h.absent |
| 245 | } |
| 246 | |
| 247 | func (h *watchedHost) done() bool { |
| 248 | switch h.mode { |
| 249 | case modeURL: |
| 250 | return h.urlLive |
| 251 | case modeRollout: |
| 252 | return h.state.final() |
| 253 | } |
| 254 | return false |
| 255 | } |
| 256 | |
| 257 | // rolloutWatch is one watch. list is the console's instance list, asked to |
| 258 | // answer by the watch's deadline; check is one URL poll (checkTarget in |
| 259 | // production); onURL runs once, the first time any host is watched by |
| 260 | // URL, so the probe-or-header note prints only when it is true. channel names |
| 261 | // the `promote --retry` a failure suggests. interval is the URL poll cadence |
| 262 | // and listEvery the list's (R5); a zero listEvery lists on every sweep. |
| 263 | type rolloutWatch struct { |
| 264 | w io.Writer |
| 265 | app, ver, channel string |
| 266 | timeout, interval time.Duration |
| 267 | listEvery time.Duration |
| 268 | list func(deadline time.Time) (watchList, error) |
| 269 | check func(t watchTarget, n int64) (bool, string) |
| 270 | onURL func() |
| 271 | diag *watchDiagnostics |
| 272 | } |
| 273 | |
| 274 | func (rw rolloutWatch) run(targets []watchTarget) error { |
| 275 | // One deadline governs the whole watch (Astra plan 6 R1 #2): every list |
| 276 | // request, every wait between lists, and what is left for watchVersion. |
| 277 | deadline := time.Now().Add(rw.timeout) |
| 278 | // A failed list (a 5xx, a rate limit, a console restart, the network) |
| 279 | // says nothing about the boxes, so it is asked again on the list's |
| 280 | // cadence until the deadline, and never read as "no reports" (Astra plan |
| 281 | // 6 R1 #4: a URL poll would wake an asleep instance on a reporting box). |
| 282 | l, err := rw.list(deadline) |
| 283 | for err != nil && !listAbsent(err) { |
| 284 | left := time.Until(deadline) |
| 285 | if left <= 0 { |
| 286 | return rw.listFailed(err) |
| 287 | } |
| 288 | time.Sleep(min(max(rw.listEvery, rw.interval), left)) |
| 289 | if time.Until(deadline) <= 0 { |
| 290 | return rw.listFailed(err) |
| 291 | } |
| 292 | l, err = rw.list(deadline) |
| 293 | } |
| 294 | lastList := time.Now() |
| 295 | // Today's watch, byte for byte (R10), only on affirmative evidence that |
| 296 | // no target is on a reporting box: the console has no list for the app |
| 297 | // (listAbsent), or every target is listed, the console read every box's |
| 298 | // report, and none of the targets has one the watch follows (legacyOnly). |
| 299 | if err != nil || legacyOnly(l, targets) { |
| 300 | if rw.onURL != nil { |
| 301 | rw.onURL() |
| 302 | } |
| 303 | return watchVersion(rw.w, targets, rw.ver, time.Until(deadline), rw.interval, rw.diag) |
| 304 | } |
| 305 | var hosts []*watchedHost |
| 306 | seen := map[string]bool{} |
| 307 | for _, t := range targets { |
| 308 | if !seen[t.Host] { |
| 309 | seen[t.Host] = true |
| 310 | hosts = append(hosts, &watchedHost{t: t, state: rolloutPending}) |
| 311 | } |
| 312 | } |
| 313 | c := colorFor(rw.w) |
| 314 | mutedf(rw.w, watchIntro, rw.ver) |
| 315 | rw.apply(hosts, l) |
| 316 | boxPrinted := map[string]bool{} |
| 317 | announcedURL := false |
| 318 | var n int64 |
| 319 | for { |
| 320 | urlPending := false |
| 321 | for _, h := range hosts { |
| 322 | if h.urlPolling() { |
| 323 | urlPending = true |
| 324 | } |
| 325 | } |
| 326 | if urlPending && !announcedURL { |
| 327 | announcedURL = true |
| 328 | if rw.onURL != nil { |
| 329 | rw.onURL() |
| 330 | } |
| 331 | } |
| 332 | var wg sync.WaitGroup |
| 333 | for _, h := range hosts { |
| 334 | if !h.urlPolling() { |
| 335 | continue |
| 336 | } |
| 337 | n++ |
| 338 | wg.Add(1) |
| 339 | go func(h *watchedHost, n int64) { |
| 340 | defer wg.Done() |
| 341 | h.urlLive, h.urlDiag = rw.check(h.t, n) |
| 342 | }(h, n) |
| 343 | } |
| 344 | wg.Wait() |
| 345 | for _, h := range hosts { |
| 346 | if h.done() && !h.printed { |
| 347 | h.printed = true |
| 348 | rw.printHost(c, h) |
| 349 | } |
| 350 | } |
| 351 | boxes, labels := groupByBox(hosts) |
| 352 | boxesDone := true |
| 353 | for _, l := range labels { |
| 354 | if !boxDone(boxes[l]) { |
| 355 | boxesDone = false |
| 356 | continue |
| 357 | } |
| 358 | if !boxPrinted[l] { |
| 359 | boxPrinted[l] = true |
| 360 | fmt.Fprintln(rw.w, boxSummary(l, boxes[l])) |
| 361 | } |
| 362 | } |
| 363 | hostsDone := true |
| 364 | for _, h := range hosts { |
| 365 | hostsDone = hostsDone && h.done() |
| 366 | } |
| 367 | if hostsDone && boxesDone { |
| 368 | return rw.result(hosts) |
| 369 | } |
| 370 | if time.Now().After(deadline) { |
| 371 | return rw.timedOut(hosts, boxes, labels) |
| 372 | } |
| 373 | time.Sleep(rw.wait(hosts, lastList, deadline)) |
| 374 | if time.Since(lastList) >= rw.listEvery { |
| 375 | l, err = rw.list(deadline) |
| 376 | lastList = time.Now() |
| 377 | // A list that failed (a rate limit, a console restart) changes |
| 378 | // nothing: every host keeps its mode and state until a list |
| 379 | // answers. |
| 380 | if err == nil { |
| 381 | rw.apply(hosts, l) |
| 382 | } |
| 383 | } |
| 384 | } |
| 385 | } |
| 386 | |
| 387 | // wait is how long the loop sleeps: the URL poll interval while any host is |
| 388 | // watched by URL, otherwise until the next list is due; never past the |
| 389 | // deadline, so the timeout report comes on time. |
| 390 | func (rw rolloutWatch) wait(hosts []*watchedHost, lastList, deadline time.Time) time.Duration { |
| 391 | d := rw.interval |
| 392 | urlPending := false |
| 393 | for _, h := range hosts { |
| 394 | urlPending = urlPending || h.urlPolling() |
| 395 | } |
| 396 | if !urlPending { |
| 397 | d = max(d, time.Until(lastList.Add(rw.listEvery))) |
| 398 | } |
| 399 | if left := time.Until(deadline); left > 0 && left < d { |
| 400 | d = left |
| 401 | } |
| 402 | return d |
| 403 | } |
| 404 | |
| 405 | // legacyOnly is the evidence the fallback to watchVersion needs (Astra plan |
| 406 | // 6 R1 #4, #7): every target is on the list, the console's rollout join read |
| 407 | // every box, and no target has a report the watch follows. It is decided on |
| 408 | // the targets alone. A list cut short for the whole app neither causes nor |
| 409 | // blocks it, and a report on an instance this deploy does not target says |
| 410 | // nothing about the targets' boxes. |
| 411 | func legacyOnly(l watchList, targets []watchTarget) bool { |
| 412 | if l.incomplete { |
| 413 | return false |
| 414 | } |
| 415 | listed := make(map[string]*apiRollout, len(l.recs)) |
| 416 | for _, r := range l.recs { |
| 417 | listed[r.Host] = r.Rollout |
| 418 | } |
| 419 | for _, t := range targets { |
| 420 | ro, ok := listed[t.Host] |
| 421 | if !ok || follows(ro) { |
| 422 | return false |
| 423 | } |
| 424 | } |
| 425 | return true |
| 426 | } |
| 427 | |
| 428 | // listFailed is the timeout when no list answered before the deadline: |
| 429 | // nothing was followed and nothing was polled, so it names the list. |
| 430 | func (rw rolloutWatch) listFailed(err error) error { |
| 431 | var b strings.Builder |
| 432 | fmt.Fprintf(&b, watchTimeout, rw.ver) |
| 433 | fmt.Fprintf(&b, "\n %s: %v", watchListFailed, err) |
| 434 | if rw.diag != nil && rw.diag.logsHint != "" { |
| 435 | fmt.Fprintf(&b, "\nfull timeline: %s", rw.diag.logsHint) |
| 436 | } |
| 437 | return errors.New(b.String()) |
| 438 | } |
| 439 | |
| 440 | // apply folds one answered list into the hosts. |
| 441 | // |
| 442 | // - A host with a report the watch follows is watched through it from now |
| 443 | // on (R4), also when it was polled by URL, even live: the report is the |
| 444 | // box's word about which build runs there, and a URL answer is not |
| 445 | // (Astra plan 6 R1 #3). Its report is refreshed every list even once it |
| 446 | // is final, because its box is done only on the box's latest scheduler |
| 447 | // word. |
| 448 | // - A final state persists only across MISSING observations: a host |
| 449 | // absent from a truncated list, a dark box. An explicit report that the |
| 450 | // host's box has another build installed takes a swapped, asleep or held |
| 451 | // host back to pending, named by what is installed (pendingDetail): the |
| 452 | // watched build no longer runs there, and Review Focus 4 forbids success |
| 453 | // for a replaced build. A failure stays failed: the build did fail. |
| 454 | // - A host missing from the list keeps its mode and state (R3), whether |
| 455 | // the cap cut it off or the console could not read its record: nothing |
| 456 | // says it uses the old path, and switching it to URL polling would wake |
| 457 | // it if it is asleep. It is also no longer polled if it was (absent), |
| 458 | // until it is listed again (Astra plan 6 R3). |
| 459 | // - A host with no followed report on a list whose rollout join was |
| 460 | // incomplete keeps its mode and state too: its box's report may be the |
| 461 | // one the console could not read (Astra plan 6 R1 #4). An undecided host |
| 462 | // stays undecided, and is named so in a timeout. |
| 463 | // - A host the watch follows whose report is gone is dark (R1): pending if |
| 464 | // it was not final, never polled. |
| 465 | // - Any other host is watched by URL, exactly as watchVersion would. |
| 466 | func (rw rolloutWatch) apply(hosts []*watchedHost, l watchList) { |
| 467 | listed := make(map[string]*apiRollout, len(l.recs)) |
| 468 | for i := range l.recs { |
| 469 | listed[l.recs[i].Host] = l.recs[i].Rollout |
| 470 | } |
| 471 | for _, h := range hosts { |
| 472 | ro, ok := listed[h.t.Host] |
| 473 | h.absent = !ok |
| 474 | switch { |
| 475 | case follows(ro): |
| 476 | if h.mode != modeRollout { |
| 477 | // What a URL answer printed ("live") stands only if the |
| 478 | // report agrees; otherwise the report's own word prints |
| 479 | // once it is final. |
| 480 | h.state, h.fin = rolloutPending, nil |
| 481 | } |
| 482 | h.mode, h.ro, h.dark, h.unread = modeRollout, ro, false, false |
| 483 | switch { |
| 484 | case !h.state.final(): |
| 485 | h.state = classifyRollout(ro, rw.ver) |
| 486 | if h.state.final() { |
| 487 | fin := *ro |
| 488 | h.fin = &fin |
| 489 | } |
| 490 | h.printed = h.printed && h.state == rolloutSwapped |
| 491 | case h.state != rolloutFailed && ro.Installed != rw.ver: |
| 492 | h.state, h.fin, h.printed = rolloutPending, nil, false |
| 493 | } |
| 494 | case !ok: |
| 495 | // Absent from this list: cut short, or a record the console could |
| 496 | // not read. Neither says the host uses the old path, so it keeps |
| 497 | // its mode and state and is never moved to URL watching here |
| 498 | // (Astra plan 6 R3). On an untruncated list the console left it |
| 499 | // out because it could not read something, which is what |
| 500 | // watchPendingUnread says. |
| 501 | h.unread = !l.truncated |
| 502 | case h.mode == modeURL && h.urlLive: |
| 503 | // Live by URL is final too, until a report says otherwise. |
| 504 | case l.incomplete: |
| 505 | h.unread = h.mode == modeUnknown |
| 506 | case h.mode == modeRollout: |
| 507 | h.dark = true |
| 508 | default: |
| 509 | h.mode = modeURL |
| 510 | } |
| 511 | } |
| 512 | } |
| 513 | |
| 514 | func groupByBox(hosts []*watchedHost) (map[string][]*watchedHost, []string) { |
| 515 | boxes := map[string][]*watchedHost{} |
| 516 | var labels []string |
| 517 | for _, h := range hosts { |
| 518 | if h.mode != modeRollout || h.ro == nil { |
| 519 | continue |
| 520 | } |
| 521 | if _, ok := boxes[h.ro.Box]; !ok { |
| 522 | labels = append(labels, h.ro.Box) |
| 523 | } |
| 524 | boxes[h.ro.Box] = append(boxes[h.ro.Box], h) |
| 525 | } |
| 526 | sort.Strings(labels) |
| 527 | return boxes, labels |
| 528 | } |
| 529 | |
| 530 | // boxDone is spec §9.1's rule: every targeted instance on the box is final, |
| 531 | // and its scheduler has nothing queued, running or settling for the app, by |
| 532 | // the latest word the watch has from the box. |
| 533 | func boxDone(hs []*watchedHost) bool { |
| 534 | for _, h := range hs { |
| 535 | if !h.state.final() || !h.ro.SchedulerIdle { |
| 536 | return false |
| 537 | } |
| 538 | } |
| 539 | return true |
| 540 | } |
| 541 | |
| 542 | func boxSummary(label string, hs []*watchedHost) string { |
| 543 | counts := map[rolloutState]int{} |
| 544 | for _, h := range hs { |
| 545 | counts[h.state]++ |
| 546 | } |
| 547 | var parts []string |
| 548 | for _, p := range []struct { |
| 549 | s rolloutState |
| 550 | format string |
| 551 | }{{rolloutSwapped, watchCountSwapped}, {rolloutAsleep, watchCountAsleep}, {rolloutFailed, watchCountFailed}, {rolloutHeld, watchCountHeld}} { |
| 552 | if counts[p.s] > 0 { |
| 553 | parts = append(parts, fmt.Sprintf(p.format, counts[p.s])) |
| 554 | } |
| 555 | } |
| 556 | return fmt.Sprintf(watchBox, label, strings.Join(parts, ", ")) |
| 557 | } |
| 558 | |
| 559 | // stayedOn is the build a held instance is still on: what it serves, or, |
| 560 | // asleep, its last good build. |
| 561 | func stayedOn(r *apiRollout) string { |
| 562 | if r.Serving != "" { |
| 563 | return r.Serving |
| 564 | } |
| 565 | return r.Good |
| 566 | } |
| 567 | |
| 568 | func (rw rolloutWatch) printHost(c colors, h *watchedHost) { |
| 569 | host := h.t.Host |
| 570 | if h.mode == modeURL { |
| 571 | fmt.Fprintf(rw.w, "%s https://%s\n", c.ok("live"), host) |
| 572 | return |
| 573 | } |
| 574 | switch h.state { |
| 575 | case rolloutSwapped: |
| 576 | fmt.Fprintf(rw.w, "%s https://%s\n", c.ok(watchWordSwapped), host) |
| 577 | case rolloutAsleep: |
| 578 | fmt.Fprintf(rw.w, "%s https://%s (%s)\n", c.ok(watchWordInstalled), host, watchAsleepTail) |
| 579 | case rolloutFailed: |
| 580 | fmt.Fprintf(rw.w, "%s https://%s: %s\n", c.warn(watchWordFailed), host, rolloutwords.Reason(h.fin.Reason)) |
| 581 | case rolloutHeld: |
| 582 | fmt.Fprintf(rw.w, "%s https://%s: %s\n", c.warn(watchWordHeld), host, fmt.Sprintf(watchHeldTail, stayedOn(h.fin), rw.ver)) |
| 583 | } |
| 584 | } |
| 585 | |
| 586 | func (rw rolloutWatch) result(hosts []*watchedHost) error { |
| 587 | failed, held, down := 0, 0, false |
| 588 | for _, h := range hosts { |
| 589 | if h.mode != modeRollout { |
| 590 | continue |
| 591 | } |
| 592 | switch h.state { |
| 593 | case rolloutFailed: |
| 594 | failed++ |
| 595 | down = down || h.fin.Attempt == status.AttemptDown |
| 596 | case rolloutHeld: |
| 597 | held++ |
| 598 | } |
| 599 | } |
| 600 | if failed == 0 && held == 0 { |
| 601 | return nil |
| 602 | } |
| 603 | msg := fmt.Sprintf(watchFailedSummary, rw.ver, failed, held, rw.app, rw.ver, rw.channel) |
| 604 | if down { |
| 605 | msg += "\n" + fmt.Sprintf(watchHoldDownSummary, rw.app) |
| 606 | } |
| 607 | return errors.New(msg) |
| 608 | } |
| 609 | |
| 610 | // pendingDetail is what a timeout says about a host still pending. |
| 611 | func (rw rolloutWatch) pendingDetail(h *watchedHost) string { |
| 612 | r := h.ro |
| 613 | switch { |
| 614 | case h.mode == modeUnknown && h.unread: |
| 615 | return watchPendingUnread |
| 616 | case h.mode == modeUnknown: |
| 617 | return watchPendingUnlisted |
| 618 | case h.mode == modeURL && h.absent && h.urlDiag == "": |
| 619 | // Dropped off the list before any poll said anything. |
| 620 | if h.unread { |
| 621 | return watchPendingUnread |
| 622 | } |
| 623 | return watchPendingUnlisted |
| 624 | case h.mode == modeURL: |
| 625 | return h.urlDiag |
| 626 | case h.dark: |
| 627 | return watchPendingDark |
| 628 | case r.Unavailable != "": |
| 629 | return rolloutwords.Unavailable(r.Unavailable) |
| 630 | case r.Installed == "": |
| 631 | return watchPendingNotInstalled |
| 632 | case r.Installed != rw.ver: |
| 633 | return fmt.Sprintf(watchPendingOther, r.Installed) |
| 634 | case r.Attempt == status.AttemptQueued: |
| 635 | return watchPendingQueued |
| 636 | case r.Attempt == status.AttemptRunning: |
| 637 | return watchPendingRunning |
| 638 | case r.Attempt == status.AttemptSettling: |
| 639 | return watchPendingSettling |
| 640 | } |
| 641 | return watchPendingWaiting |
| 642 | } |
| 643 | |
| 644 | func (rw rolloutWatch) timedOut(hosts []*watchedHost, boxes map[string][]*watchedHost, labels []string) error { |
| 645 | var b strings.Builder |
| 646 | fmt.Fprintf(&b, watchTimeout, rw.ver) |
| 647 | var pending []*watchedHost |
| 648 | for _, h := range hosts { |
| 649 | if !h.done() { |
| 650 | pending = append(pending, h) |
| 651 | } |
| 652 | } |
| 653 | sort.Slice(pending, func(i, j int) bool { return pending[i].t.Host < pending[j].t.Host }) |
| 654 | for _, h := range pending { |
| 655 | fmt.Fprintf(&b, "\n %s: %s", h.t.Host, rw.pendingDetail(h)) |
| 656 | // The reason a wake failed, from the platform log stream, as |
| 657 | // watchVersion gives it: only for a host polled by URL, whose poll |
| 658 | // is what woke it. |
| 659 | if h.mode == modeURL && rw.diag != nil && rw.diag.wakeReason != nil { |
| 660 | if r := rw.diag.wakeReason(h.t.Host); r != "" { |
| 661 | fmt.Fprintf(&b, "\n %s", r) |
| 662 | } |
| 663 | } |
| 664 | } |
| 665 | for _, l := range labels { |
| 666 | allFinal := true |
| 667 | for _, h := range boxes[l] { |
| 668 | allFinal = allFinal && h.state.final() |
| 669 | } |
| 670 | if allFinal && !boxDone(boxes[l]) { |
| 671 | fmt.Fprintf(&b, "\n "+watchBoxBusy, l) |
| 672 | } |
| 673 | } |
| 674 | if rw.diag != nil && rw.diag.logsHint != "" { |
| 675 | fmt.Fprintf(&b, "\nfull timeline: %s", rw.diag.logsHint) |
| 676 | } |
| 677 | return errors.New(b.String()) |
| 678 | } |
| 679 | |
| 680 | // watchDeployed is the watch for an instance (non-static) deploy of ver onto |
| 681 | // channel. |
| 682 | func watchDeployed(ctx context.Context, sess *apiSession, acct apiAcct, app, ver, channel string, hosts []string, probe string, timeout time.Duration) error { |
| 683 | targets := make([]watchTarget, 0, len(hosts)) |
| 684 | for _, h := range hosts { |
| 685 | targets = append(targets, watchTarget{Host: h, Origin: watchOrigin(h), Probe: probe}) |
| 686 | } |
| 687 | return newDeployWatch(ctx, sess, acct, app, ver, channel, probe, timeout).run(targets) |
| 688 | } |
| 689 | |
| 690 | // newDeployWatch is watchDeployed's watch, built apart so a test can read |
| 691 | // its cadence without a console. |
| 692 | func newDeployWatch(ctx context.Context, sess *apiSession, acct apiAcct, app, ver, channel, probe string, timeout time.Duration) rolloutWatch { |
| 693 | client := &http.Client{Timeout: 10 * time.Second} |
| 694 | return rolloutWatch{ |
| 695 | w: stdout, app: app, ver: ver, channel: channel, timeout: timeout, |
| 696 | interval: deployWatchPollEvery, listEvery: deployWatchListEvery, |
| 697 | list: func(deadline time.Time) (watchList, error) { |
| 698 | lctx, cancel := context.WithDeadline(ctx, deadline) |
| 699 | defer cancel() |
| 700 | return sess.watchInstanceList(lctx, acct, app) |
| 701 | }, |
| 702 | check: func(t watchTarget, n int64) (bool, string) { return checkTarget(client, t, ver, n) }, |
| 703 | onURL: func() { printWatchNote(probe, false, app) }, |
| 704 | diag: deployDiagnostics(ctx, sess, acct, app), |
| 705 | } |
| 706 | } |
| 707 | |
| 708 | // deployDiagnostics is the timeout report's help, moved out of runDeploy |
| 709 | // unchanged. |
| 710 | func deployDiagnostics(ctx context.Context, sess *apiSession, acct apiAcct, app string) *watchDiagnostics { |
| 711 | return &watchDiagnostics{ |
| 712 | // A timed-out host's wake reason, from the platform log stream — |
| 713 | // scoped tight (one host, errors only, the watch's own recent past) |
| 714 | // so a diagnosis this may print was caused by THIS deploy, not an |
| 715 | // incident from earlier in the day. |
| 716 | wakeReason: func(host string) string { |
| 717 | q := url.Values{"stream": {"platform"}, "level": {"error"}, |
| 718 | "host": {host}, "since": {"15m"}, "limit": {"1"}} |
| 719 | res, err := sess.logsGet(ctx, acct, app, q) |
| 720 | if err != nil || len(res.Records) == 0 { |
| 721 | return "" |
| 722 | } |
| 723 | return res.Records[len(res.Records)-1].Msg |
| 724 | }, |
| 725 | logsHint: fmt.Sprintf("carlos logs --app %s --account %s --stream platform", app, acct.Seg), |
| 726 | } |
| 727 | } |
| 728 | |
| 729 | // printWatchNote is runDeploy's probe-or-header note, moved out unchanged. |
| 730 | func printWatchNote(probe string, staticDeploy bool, app string) { |
| 731 | if probe != "" { |
| 732 | mutedf(stdout, "watching %s for the build %s reports itself\n", probe, app) |
| 733 | return |
| 734 | } |
| 735 | // Said out loud, every time, because the silence is what the bug was |
| 736 | // made of (discussion #13). With no probe this watch can only ask |
| 737 | // the ROUTER what release was adopted; it cannot ask the process |
| 738 | // what it is running. A deploy that goes green here has proved that |
| 739 | // adoption happened, not that anything restarted — and an app whose |
| 740 | // instance failed to cycle looks identical. It is a one-line fix |
| 741 | // per app, so the line names it rather than describing the problem |
| 742 | // and leaving the reader to find the verb. |
| 743 | mutedf(stdout, "watching the X-Carlos-Version header: it reports the release ADOPTED, not the build running\n") |
| 744 | // The advisory below assumes a process to probe. A static route has |
| 745 | // none — no /api/version, no exec child to cycle — so the suggested |
| 746 | // command would only teach the operator to configure a probe that |
| 747 | // 404s, which then fails and times out every future `carlos deploy` |
| 748 | // for that site. Static deploys get the line above (still true: the |
| 749 | // header reports adoption, not a running build) and stop there. |
| 750 | if !staticDeploy { |
| 751 | mutedf(stdout, " a probe proves the process actually cycled — carlos apps build-probe --app %s /api/version\n", app) |
| 752 | } |
| 753 | } |
| 754 | |
amadan