| 1 | package host |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "log" |
| 7 | "sync" |
| 8 | "sync/atomic" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/activator" |
| 12 | "github.com/carlosframework/platform/internal/appconfig" |
| 13 | "github.com/carlosframework/platform/internal/registry" |
| 14 | "github.com/carlosframework/platform/internal/release" |
| 15 | "github.com/carlosframework/platform/internal/rollout" |
| 16 | "github.com/carlosframework/platform/internal/status" |
| 17 | ) |
| 18 | |
| 19 | // rolloutBookHonoured is RolloutReport.FailureBook's code for a box that |
| 20 | // declined CARLOS_SWAP=overlap and honours the failure records an earlier |
| 21 | // armed run left (honourDeclinedFailures). A code, not member-facing text. |
| 22 | const rolloutBookHonoured = "honoured" |
| 23 | |
| 24 | // refuseNoSwaps is the wake refusal for a managed instance whose build needs |
| 25 | // the scheduler this box cannot run: refuseManaged's text for a reason it has |
| 26 | // no words for, and a declined box's for a build a failure record holds. |
| 27 | const refuseNoSwaps = "this box cannot run two builds side by side, which this instance's build needs" |
| 28 | |
| 29 | // failureHolds is the failure book a box that declined CARLOS_SWAP=overlap |
| 30 | // still honours (final review F6, which replaced PF15's "today's paths"). An |
| 31 | // earlier armed run recorded these failures, and losing the delegation the |
| 32 | // scheduler needs is not an operator clearing them: a hold build whose |
| 33 | // migration failed must not run again on the live link. So, with generations |
| 34 | // off and no scheduler, the box keeps today's paths except that a managed |
| 35 | // wake onto a build the book holds for that instance is refused (Planner), |
| 36 | // and refused again at the real launch, against the build that launch runs |
| 37 | // (gate, Task 9); and the converge leg, the adopt bounce, the restart leg and |
| 38 | // config sync's restart leave such an instance running rather than stop it |
| 39 | // into a refused wake. The book holds records and hold-ran markers. |
| 40 | // |
| 41 | // The book is read once, at arming, and nothing on a declined box records a |
| 42 | // failure or a marker, or prunes (Astra R2 ruling 194, "startup-only |
| 43 | // records"). But it is cleared while the agent runs, as an armed box's is |
| 44 | // (Task 10): a verified signed `carlos instances retry` (retryHost), a |
| 45 | // `carlos promote --retry` pointer and a signed rollback pointer |
| 46 | // (applyPointers) each clear in this one book and save it. The planner, the |
| 47 | // launch gate and the legs read it live, so a clear lets the instance start |
| 48 | // at once, with no agent restart. It is also written at arming when it owes |
| 49 | // a save (an armed run left intents in it, or an earlier agent wrote it), |
| 50 | // and by the periodic pass while that save keeps failing (pass; the |
| 51 | // crash-epoch hard limit, rule 6). |
| 52 | type failureHolds struct { |
| 53 | book *rollout.FailureBook |
| 54 | // get reads a route by host, for the legs that are handed only a host. |
| 55 | get func(host string) (*registry.Route, error) |
| 56 | // list reads every route, for the box's status report. |
| 57 | list func() ([]registry.Route, error) |
| 58 | // optDir is where the live links are: a wake execs the build |
| 59 | // live/<host> names, which root repoints before it stamps the row. |
| 60 | optDir string |
| 61 | // said is the (leg, host, build) triples already logged, so a leg that |
| 62 | // runs every two seconds says once that it is leaving a host alone. |
| 63 | said sync.Map |
| 64 | } |
| 65 | |
| 66 | // declinedHolds is set by honourDeclinedFailures, before any leg is built, |
| 67 | // and only when the book holds at least one record or hold-ran marker. Nil: |
| 68 | // the flag is off, armed, refusing, or declined with an empty book, and |
| 69 | // every leg is as its own branch says. |
| 70 | var declinedHolds atomic.Pointer[failureHolds] |
| 71 | |
| 72 | // heldHolds is declinedHolds' value. |
| 73 | func heldHolds() *failureHolds { return declinedHolds.Load() } |
| 74 | |
| 75 | // honourDeclinedFailures is armRollout's branch for a box that declined the |
| 76 | // flag (spec §§6.4, 8.4; final review F6). An absent book, or one with no |
| 77 | // records and no hold-ran markers, leaves everything exactly as today: no |
| 78 | // hook, no leg changed, LiveExecGate nil. An unreadable one fails closed as |
| 79 | // refuseManaged does for an armed box, since any record in it could be a |
| 80 | // hold-down. A readable one with records or markers is honoured, read-only, |
| 81 | // through the Planner, the launch gate and the legs (failureHolds). |
| 82 | func honourDeclinedFailures(svc *activator.Service, reg *registry.Registry, dd, optDir string) { |
| 83 | book, err := rollout.OpenFailureBook(dd) |
| 84 | if err != nil { |
| 85 | log.Printf("agent: %s=overlap declined, and the failure book is unreadable: %v", rollout.SwapEnv, err) |
| 86 | refuseManaged(svc, rolloutBookUnreadable) |
| 87 | return |
| 88 | } |
| 89 | if n := book.Reconciled(); n > 0 { |
| 90 | // An armed run's executions left unrecorded (Task 5): their records |
| 91 | // are honoured below whether or not the save below lands, since the |
| 92 | // book in memory holds them either way, and the next restart |
| 93 | // reconciles the intents again if it did not. |
| 94 | log.Printf("agent: %s=overlap declined: %d execution(s) an armed run left unrecorded are now failure records", rollout.SwapEnv, n) |
| 95 | } |
| 96 | // Records or hold-ran markers (Task 9): a marker alone still bars the good |
| 97 | // build from an instance whose database a hold build may have migrated. |
| 98 | recs, marks := len(book.List()), book.HoldRanCount() |
| 99 | if recs == 0 && marks == 0 { |
| 100 | return |
| 101 | } |
| 102 | // A book that owes a save is saved now, whatever reconciliation found |
| 103 | // (the crash-epoch hard limit, rule 6): reconciled records, a new |
| 104 | // identity for a book an earlier agent wrote (rule 4), epochs given to |
| 105 | // older entries. Until it lands, status publishes no generation and no |
| 106 | // retry grant clears anything; the periodic pass (pass) retries it. A |
| 107 | // book this rule wrote and the open did not change owes nothing, and is |
| 108 | // not written. |
| 109 | if err := book.SaveIfDirty(); err != nil { |
| 110 | log.Printf("agent: %s=overlap declined: save the failure book after opening it: %v (retried by the periodic pass)", rollout.SwapEnv, err) |
| 111 | } |
| 112 | h := &failureHolds{book: book, get: reg.Get, list: reg.List, optDir: optDir} |
| 113 | declinedHolds.Store(h) |
| 114 | svc.Planner = h.plan |
| 115 | // And at the real launch, with the build actually started (Task 9): the |
| 116 | // plan reads the live link before the lease and the restore, and root |
| 117 | // can repoint it at a held build in between. |
| 118 | svc.SetLiveExecGate(h.gate) |
| 119 | log.Printf("agent: %s=overlap declined, but %d failure record(s) and %d hold-ran marker(s) from an armed run are honoured: "+ |
| 120 | "no managed instance wakes, starts, or is stopped, onto a build they hold for it", rollout.SwapEnv, recs, marks) |
| 121 | } |
| 122 | |
| 123 | // refusal is the wake refusal for starting build on rt, "" when it may |
| 124 | // start. Only a managed route can be held: the scheduler records nothing |
| 125 | // else. |
| 126 | // |
| 127 | // The rule is the armed planner's (rollout.decideWake and decideHoldRan, |
| 128 | // spec §8.5, §6.4), with the one difference that a declined box cannot boot |
| 129 | // the good build instead, so it refuses where the armed box would recover. |
| 130 | // A hold-down holds whatever the instance served, in that build's words. |
| 131 | // Then, with a hold-ran marker of rt's app standing for the instance (Task |
| 132 | // 9; Astra R3 N5/N7, ruling 205): |
| 133 | // - the marked build may start, unless the instance has a good build and |
| 134 | // a record holds the marked build for it. A record never holds a build |
| 135 | // the instance served, so a marker naming the good build itself lets it |
| 136 | // start (the marked-build rule), whether the stamp or the live link |
| 137 | // names it: unlike the armed planner, a declined box never boots the |
| 138 | // good build as a recovery, so the build being started is always the |
| 139 | // installed one, stamped or linked (fix round 1, ruling 4a); |
| 140 | // - otherwise the good build is refused: an installed good build is no |
| 141 | // proof of a signed rollback, since root's automatic sibling repoint |
| 142 | // installs it too, and only the pointer pass clears the marker on one |
| 143 | // (Task 10); |
| 144 | // - any other build a record holds for the instance is refused. |
| 145 | // |
| 146 | // All three refuse in HeldDownRefusal's words for the marked build, as the |
| 147 | // armed planner does, so "roll back or retry" stays true here. Without a |
| 148 | // marker, a plain record holds a build only when the instance has a good |
| 149 | // build to fall back to (a build it has served is never held by a record |
| 150 | // alone); an instance with none boots its installed build as today, because |
| 151 | // refusing it would turn one sibling's failure into an outage for every |
| 152 | // sleeping sibling. It reads only memory, so the activator may ask it under |
| 153 | // its own lock (gate). |
| 154 | func (h *failureHolds) refusal(rt registry.Route, build string) string { |
| 155 | return h.holdOf(rt, build).refuse |
| 156 | } |
| 157 | |
| 158 | // hold is why the book holds an instance off one build: the wake refusal, |
| 159 | // the build whose restriction refuses it (by), which is what a retry of |
| 160 | // the instance must name (RetryHostAt), and whether the instance stays down |
| 161 | // for it (a hold-down, or a hold-ran marker) rather than on what it runs. |
| 162 | type hold struct { |
| 163 | refuse, by string |
| 164 | down bool |
| 165 | } |
| 166 | |
| 167 | // holdOf is refusal with the build it holds by: the hold-down's build, the |
| 168 | // marker's build when the marker refuses the marked build or the good |
| 169 | // build, and otherwise the build a record holds, which a marker may refuse |
| 170 | // in its own words while only that record's retry lifts it. |
| 171 | // |
| 172 | // Eligibility is rollout.Restrictable, not Managed (final review round 1, |
| 173 | // finding 2): a row a move holds (landing, restoring) is the row a moves |
| 174 | // probe or landing launches, and the book's restriction of the instance |
| 175 | // applies there as on its ready row. |
| 176 | func (h *failureHolds) holdOf(rt registry.Route, build string) hold { |
| 177 | if !rollout.Restrictable(rt) || build == "" { |
| 178 | return hold{} |
| 179 | } |
| 180 | acct, app, host := release.RouteAccount(rt), rt.App, rt.Host |
| 181 | hasGood := rt.GoodVersion != "" |
| 182 | if h.book.HeldDown(acct, app, build, host) { |
| 183 | return hold{rollout.HeldDownRefusal(build), build, true} |
| 184 | } |
| 185 | if mark, ok := h.book.HoldRanOn(host); ok && mark.Account == acct && mark.App == app { |
| 186 | isGood := hasGood && build == rt.GoodVersion |
| 187 | switch { |
| 188 | case build == mark.Build: |
| 189 | if hasGood && h.book.Blocks(acct, app, build, host, isGood) { |
| 190 | return hold{rollout.HeldDownRefusal(mark.Build), mark.Build, true} |
| 191 | } |
| 192 | return hold{} |
| 193 | case isGood: |
| 194 | return hold{rollout.HeldDownRefusal(mark.Build), mark.Build, true} |
| 195 | case hasGood && h.book.Blocks(acct, app, build, host, false): |
| 196 | return hold{rollout.HeldDownRefusal(mark.Build), build, true} |
| 197 | } |
| 198 | return hold{} |
| 199 | } |
| 200 | if hasGood && h.book.Blocks(acct, app, build, host, build == rt.GoodVersion) { |
| 201 | return hold{refuseNoSwaps, build, false} |
| 202 | } |
| 203 | return hold{} |
| 204 | } |
| 205 | |
| 206 | // heldBuild is the build the book holds rt off and its hold (holdOf), "" |
| 207 | // when nothing holds it. Both the installed build (root's stamp) and the |
| 208 | // build live/<host> names are asked, because a wake execs the live link and |
| 209 | // root repoints it before it stamps the row; an unreadable link is left to |
| 210 | // the stamp. |
| 211 | func (h *failureHolds) heldBuild(rt registry.Route) (string, hold) { |
| 212 | if !rollout.Restrictable(rt) { |
| 213 | return "", hold{} |
| 214 | } |
| 215 | builds := []string{rt.Version} |
| 216 | if live, err := release.LiveVersion(h.optDir, rt.Host); err == nil && live != "" && live != rt.Version { |
| 217 | builds = append(builds, live) |
| 218 | } |
| 219 | for _, b := range builds { |
| 220 | if hd := h.holdOf(rt, b); hd.refuse != "" { |
| 221 | return b, hd |
| 222 | } |
| 223 | } |
| 224 | return "", hold{} |
| 225 | } |
| 226 | |
| 227 | // plan is the declined box's Planner: a managed wake onto a held build is |
| 228 | // refused with fixed text that names the build at most, never an error; any |
| 229 | // other wake gets the zero plan, which is today's wake from the live link. |
| 230 | // rt is the row doWake has just re-read (Service.Routes). |
| 231 | func (h *failureHolds) plan(_ context.Context, rt registry.Route) activator.WakePlan { |
| 232 | _, hd := h.heldBuild(rt) |
| 233 | return activator.WakePlan{Refuse: hd.refuse} |
| 234 | } |
| 235 | |
| 236 | // report is the declined box's per-host rollout status (Task 10, plan |
| 237 | // review #5): each managed instance the book holds, with the build that |
| 238 | // holds it (HeldBy) and that restriction's generation as published |
| 239 | // (HeldEpoch, rollout.FailureBook.PublishedEpoch: 0 while the book owes a |
| 240 | // save), which the console's retry doors sign into their grant, attempt |
| 241 | // down when it stays down for the hold and failed otherwise. Nil when |
| 242 | // nothing is held or the routes cannot be read. A declined box runs no |
| 243 | // scheduler, so it reports no serving build and no attempt in progress. |
| 244 | func (h *failureHolds) report() map[string]status.HostRollout { |
| 245 | routes, err := h.list() |
| 246 | if err != nil { |
| 247 | return nil |
| 248 | } |
| 249 | var out map[string]status.HostRollout |
| 250 | for _, rt := range routes { |
| 251 | build, hd := h.heldBuild(rt) |
| 252 | if build == "" { |
| 253 | continue |
| 254 | } |
| 255 | acct := release.RouteAccount(rt) |
| 256 | attempt := status.AttemptFailed |
| 257 | if hd.down { |
| 258 | attempt = status.AttemptDown |
| 259 | } |
| 260 | if out == nil { |
| 261 | out = map[string]status.HostRollout{} |
| 262 | } |
| 263 | out[rt.Host] = status.HostRollout{Account: acct, App: rt.App, Installed: rt.Version, Good: rt.GoodVersion, |
| 264 | Attempt: attempt, AttemptBuild: hd.by, HeldBy: hd.by, HeldEpoch: h.book.PublishedEpoch(acct, rt.App, hd.by, rt.Host), |
| 265 | HeldBook: h.book.ID(), // the book the epoch counts in (final review round 1, finding 3) |
| 266 | RetryV: status.HostRetryV} // grants ride the host-retry log (readiness round 2, Task 3) |
| 267 | } |
| 268 | return out |
| 269 | } |
| 270 | |
| 271 | // retryHost is restartMemory.retryHost on a declined box (Task 10, ruling |
| 272 | // 194): a `carlos instances retry` whose root-signed grant the restart leg |
| 273 | // has verified, applied to this box's book exactly as the armed box applies |
| 274 | // it (RetryHostAt): while build's restriction of host is still at epoch, in |
| 275 | // this box's book (book, final review round 1, finding 3), it |
| 276 | // clears build's record for host and host's marker when the marker names |
| 277 | // build, and saves. A grant for another build, or from before the |
| 278 | // restriction last changed, clears nothing. There is no scheduler to run a |
| 279 | // canary: the next wake, or the converge leg, starts the build as today. |
| 280 | func (h *failureHolds) retryHost(host, build, book string, epoch uint64) bool { |
| 281 | rt, err := h.get(host) |
| 282 | // Restrictable, not Managed (finding 2): a grant that arrives while a |
| 283 | // move holds the row (draining, moving, landing) is applied, not read |
| 284 | // once by the restart leg and dropped. |
| 285 | if err != nil || rt == nil || !rollout.Restrictable(*rt) || build == "" { |
| 286 | return false |
| 287 | } |
| 288 | acct := release.RouteAccount(*rt) |
| 289 | rec, mark, now, err := h.book.RetryHostAt(acct, rt.App, build, host, book, epoch) |
| 290 | if err != nil && (rec || mark) { |
| 291 | log.Printf("agent: retry %s on %s: save the failure book: %v (cleared in memory)", build, host, err) |
| 292 | } else if err != nil { |
| 293 | // The book owed a save and it failed again: nothing is cleared until |
| 294 | // one lands (the crash-epoch hard limit, rule 2). |
| 295 | log.Printf("agent: retry %s on %s: the failure book cannot be saved (%v), so nothing was cleared", build, host, err) |
| 296 | } else if !rec && !mark && now != epoch { |
| 297 | log.Printf("agent: retry %s on %s: the grant is for epoch %d, the restriction's is %d, so nothing was cleared", build, host, epoch, now) |
| 298 | } else if !rec && !mark && book != h.book.ID() { |
| 299 | log.Printf("agent: retry %s on %s: the grant is for book %q, not this box's, so nothing was cleared", build, host, book) |
| 300 | } |
| 301 | if rec || mark { |
| 302 | nudgeStatus() |
| 303 | } |
| 304 | return rec || mark |
| 305 | } |
| 306 | |
| 307 | // applyPointers is the declined box's pointer pass (Task 10, ruling 194; |
| 308 | // Correction 3): the armed box's two pointer rules (rollout.ApplyPointers) |
| 309 | // on this box's book, for every app with a record or a marker a signed |
| 310 | // rollback could clear. read is release.Channel with the agent's store and |
| 311 | // root key. The caller paces it (pointerLoop). |
| 312 | func (h *failureHolds) applyPointers(ctx context.Context, read rollout.PointerRead, routes []registry.Route) { |
| 313 | if rollout.ApplyPointers(ctx, h.book, read, routes, log.Printf) { |
| 314 | nudgeStatus() |
| 315 | } |
| 316 | } |
| 317 | |
| 318 | // pass is one turn of the declined box's periodic pass: the save the book |
| 319 | // owes, if any (the crash-epoch hard limit, rule 6), so a box whose disk |
| 320 | // recovers publishes its generations again with no pointer or grant |
| 321 | // arriving, then the pointer pass over the registry's routes. |
| 322 | func (h *failureHolds) pass(ctx context.Context, read rollout.PointerRead) { |
| 323 | if h.book.SaveOwed() { |
| 324 | if err := h.book.SaveIfDirty(); err != nil { |
| 325 | log.Printf("agent: declined pass: save the failure book: %v", err) |
| 326 | } else { |
| 327 | nudgeStatus() |
| 328 | } |
| 329 | } |
| 330 | if routes, err := h.list(); err != nil { |
| 331 | log.Printf("agent: declined pointer pass: list routes: %v", err) |
| 332 | } else { |
| 333 | h.applyPointers(ctx, read, routes) |
| 334 | } |
| 335 | } |
| 336 | |
| 337 | // pointerLoop runs pass, at most every rollout.PointerReadEvery, for the |
| 338 | // life of the agent. An app with nothing the pointer pass could clear costs |
| 339 | // no read. |
| 340 | func (h *failureHolds) pointerLoop(ctx context.Context, read rollout.PointerRead) { |
| 341 | for { |
| 342 | h.pass(ctx, read) |
| 343 | select { |
| 344 | case <-ctx.Done(): |
| 345 | return |
| 346 | case <-time.After(rollout.PointerReadEvery): |
| 347 | } |
| 348 | } |
| 349 | } |
| 350 | |
| 351 | // rolloutRetryHost is restartMemory.retryHost for this box: the armed |
| 352 | // manager's, a declined box's that honours its book (failureHolds.retryHost, |
| 353 | // Task 10), or nil, which drops every retry because nothing is held. |
| 354 | func rolloutRetryHost() func(host, build, book string, epoch uint64) bool { |
| 355 | if m := armedRollout(); m != nil { |
| 356 | return m.RetryHost |
| 357 | } |
| 358 | if h := heldHolds(); h != nil { |
| 359 | return h.retryHost |
| 360 | } |
| 361 | return nil |
| 362 | } |
| 363 | |
| 364 | // gate is the declined box's LiveExecGate (Task 9; Astra R1 #6, the R2/R3 |
| 365 | // declined residuals): the plan's refusal asked again at the real launch, |
| 366 | // about the build that launch is about to run, which root may have |
| 367 | // repointed the live link to since the plan. The backend asks it at a flat |
| 368 | // or nested instance's live-link start and a worker's spawn from the live |
| 369 | // link, the activator before a worker's converge or a recycle's move onto |
| 370 | // the live link, and a claims-backed wake once its snapshot has resolved the |
| 371 | // link (claimServingLaunchSource). An empty build (a link outside the |
| 372 | // version store, or one that could not be read) is left to the stamp, as |
| 373 | // heldBuild leaves it. It reads only memory and rt, and writes nothing: the |
| 374 | // activator asks it under its own lock. |
| 375 | func (h *failureHolds) gate(rt registry.Route, build string) error { |
| 376 | if build == "" { |
| 377 | build = rt.Version |
| 378 | } |
| 379 | if r := h.refusal(rt, build); r != "" { |
| 380 | return errors.New(r) |
| 381 | } |
| 382 | return nil |
| 383 | } |
| 384 | |
| 385 | // heldRoute reports whether rt is held, logging it once per leg and build. |
| 386 | func (h *failureHolds) heldRoute(leg string, rt registry.Route) bool { |
| 387 | build, _ := h.heldBuild(rt) |
| 388 | if build == "" { |
| 389 | return false |
| 390 | } |
| 391 | if _, said := h.said.LoadOrStore(leg+"\x00"+rt.Host+"\x00"+build, true); !said { |
| 392 | log.Printf("agent: %s: leaving %s running: a failure record or hold-ran marker from an armed run holds build %s for it, and %s=overlap is declined on this box", |
| 393 | leg, rt.Host, build, rollout.SwapEnv) |
| 394 | } |
| 395 | return true |
| 396 | } |
| 397 | |
| 398 | // heldHost is heldRoute for a leg handed only a host. A row that cannot be |
| 399 | // read counts as held: the leg leaves the instance running, which is what |
| 400 | // makeRestartRoute does with such a row anyway. |
| 401 | func (h *failureHolds) heldHost(leg, host string) bool { |
| 402 | rt, err := h.get(host) |
| 403 | if err != nil { |
| 404 | return true |
| 405 | } |
| 406 | return rt != nil && h.heldRoute(leg, *rt) |
| 407 | } |
| 408 | |
| 409 | // skipGet wraps the converge leg's row read: a held row reads as no row, so |
| 410 | // convergeAdopted passes the host by. |
| 411 | func (h *failureHolds) skipGet(get func(string) (*registry.Route, error)) func(string) (*registry.Route, error) { |
| 412 | return func(host string) (*registry.Route, error) { |
| 413 | rt, err := get(host) |
| 414 | if err == nil && rt != nil && h.heldRoute("converge", *rt) { |
| 415 | return nil, nil |
| 416 | } |
| 417 | return rt, err |
| 418 | } |
| 419 | } |
| 420 | |
| 421 | // skipHost wraps a bounce handed only a host (the adopt bounce): a held host |
| 422 | // is not stopped, which its caller reads as nothing running to cycle. |
| 423 | func (h *failureHolds) skipHost(leg string, bounce func(string) bool) func(string) bool { |
| 424 | return func(host string) bool { |
| 425 | if h.heldHost(leg, host) { |
| 426 | return false |
| 427 | } |
| 428 | return bounce(host) |
| 429 | } |
| 430 | } |
| 431 | |
| 432 | // skipRoutes wraps the restart leg's bounce over the routes it was handed, |
| 433 | // so it reads no row of its own. |
| 434 | func (h *failureHolds) skipRoutes(routes []registry.Route, bounce func(string) bool) func(string) bool { |
| 435 | if bounce == nil { |
| 436 | return nil |
| 437 | } |
| 438 | byHost := make(map[string]registry.Route, len(routes)) |
| 439 | for _, rt := range routes { |
| 440 | byHost[rt.Host] = rt |
| 441 | } |
| 442 | return func(host string) bool { |
| 443 | if rt, ok := byHost[host]; ok && h.heldRoute("restart", rt) { |
| 444 | return false |
| 445 | } |
| 446 | return bounce(host) |
| 447 | } |
| 448 | } |
| 449 | |
| 450 | // configCallbacks is config sync's callbacks on a declined box that honours |
| 451 | // records: today's restart and ack for every host but a held one, which is |
| 452 | // not stopped into a refused wake. Its ack must not claim a restart: an |
| 453 | // awake held instance still runs its old config, so the restart answers |
| 454 | // rolloutRestartRefused (the ack's RestartError); a sleeping one reads the |
| 455 | // file when it next wakes, so its ack says delivered on wake. |
| 456 | func (h *failureHolds) configCallbacks(ack func(appconfig.ConfigAck), legs configLegs) appconfig.Callbacks { |
| 457 | restart := func(host string) error { |
| 458 | if h.heldHost("config sync", host) { |
| 459 | if legs.awake != nil && legs.awake(host) { |
| 460 | return errors.New(rolloutRestartRefused) |
| 461 | } |
| 462 | return appconfig.ErrDeliveredOnWake |
| 463 | } |
| 464 | if legs.restart == nil { |
| 465 | return nil |
| 466 | } |
| 467 | return legs.restart(host) |
| 468 | } |
| 469 | return appconfig.Callbacks{Restart: restart, Ack: ack} |
| 470 | } |
| 471 | |
| 472 | // declinedHoldsStay is fastConvergeLoop's one line, at arming, when a box |
| 473 | // that honours its failure book has no deploy store or no release key (Task |
| 474 | // 10 rule 2): with neither a pointer can be read nor a retry's grant |
| 475 | // verified, so no pointer pass runs and no retry is read, and what the book |
| 476 | // holds stays held until the agent restarts with both. |
| 477 | func declinedHoldsStay(why string) { |
| 478 | if heldHolds() == nil { |
| 479 | return |
| 480 | } |
| 481 | log.Printf("agent: %s=overlap declined: %s, so no signed retry or channel pointer can clear the failure records and hold-ran markers honoured here", |
| 482 | rollout.SwapEnv, why) |
| 483 | } |
| 484 | |
amadan