| 1 | package rollout |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "time" |
| 8 | |
| 9 | "github.com/carlosframework/platform/internal/activator" |
| 10 | "github.com/carlosframework/platform/internal/registry" |
| 11 | "github.com/carlosframework/platform/internal/release" |
| 12 | ) |
| 13 | |
| 14 | // wakeBook is the failure book's read half the planner needs. |
| 15 | type wakeBook interface { |
| 16 | Blocks(acct, app, build, host string, served bool) bool |
| 17 | HeldDown(acct, app, build, host string) bool |
| 18 | HoldRanOn(host string) (HoldRan, bool) |
| 19 | } |
| 20 | |
| 21 | // holdRanFor is host's hold-ran marker when it is one of (acct, app)'s: a |
| 22 | // marker left by another app's route under the same host name says nothing |
| 23 | // about this one's database. |
| 24 | func holdRanFor(b wakeBook, acct, app, host string) (HoldRan, bool) { |
| 25 | h, ok := b.HoldRanOn(host) |
| 26 | if !ok || h.Account != acct || h.App != app { |
| 27 | return HoldRan{}, false |
| 28 | } |
| 29 | return h, true |
| 30 | } |
| 31 | |
| 32 | type wakeInputs struct { |
| 33 | Route registry.Route |
| 34 | OptDir string |
| 35 | Policy AppPolicy |
| 36 | Book wakeBook |
| 37 | // Demand is the canonical check for an adoption (spec §4.2), returning a |
| 38 | // digest always. An error is a staging refusal: the staged file is |
| 39 | // missing or is not the release's artifact. |
| 40 | Demand func(version, stagedPath string) (demand, digest string, err error) |
| 41 | } |
| 42 | |
| 43 | // HeldDownRefusal is the wake refusal for an instance where build failed in |
| 44 | // hold mode (spec §6.4, §8.3). It reaches the member's "wake failed" event, |
| 45 | // so it names the build only. A box that declined CARLOS_SWAP=overlap but |
| 46 | // honours the failure records refuses with the same words (internal/host). |
| 47 | func HeldDownRefusal(build string) string { |
| 48 | return fmt.Sprintf("build %s failed in hold mode on this instance, so it stays down until you roll back or retry the build", build) |
| 49 | } |
| 50 | |
| 51 | // BookUnreadableRefusal is the wake refusal when this box's failure records |
| 52 | // cannot be read or written: an unreadable book at arming (internal/host's |
| 53 | // refuseTexts), or a planned first wake whose intent could not be saved |
| 54 | // (Task 5). Approved text, shared rather than copied. |
| 55 | const BookUnreadableRefusal = "this box cannot start instances right now: its record of failed builds cannot be read" |
| 56 | |
| 57 | // decideWake is spec §8.2's "when the wake picks a build", as a pure |
| 58 | // function. The installed build (root's stamp) boots unless a failure record |
| 59 | // holds it for this instance, in which case the good build boots from its |
| 60 | // staged path with its retained identity, and nothing is repointed. A build |
| 61 | // this instance has never served is a first wake; in overlap mode it carries |
| 62 | // the good build as its fallback. A build that failed in hold mode on this |
| 63 | // instance refuses the wake: nothing falls back automatically once a hold |
| 64 | // build has run against the database (spec §6.4, §8.3). An instance with no |
| 65 | // good build is never refused by a record alone (spec §8.5). An error is a |
| 66 | // staging refusal for the caller to report. |
| 67 | func decideWake(in wakeInputs) (activator.WakePlan, error) { |
| 68 | rt := in.Route |
| 69 | acct, app, host := release.RouteAccount(rt), rt.App, rt.Host |
| 70 | installed := rt.Version |
| 71 | if installed == "" { |
| 72 | // Plan installs a cold host first; a plan for a host with nothing |
| 73 | // installed refuses rather than let the live link run unplanned. |
| 74 | return activator.WakePlan{Refuse: "no build is installed for this instance yet"}, nil |
| 75 | } |
| 76 | hasGood := rt.GoodVersion != "" && rt.GoodDigest != "" |
| 77 | good := release.StagedIdentity{Version: rt.GoodVersion, StagedPath: release.VersionPath(in.OptDir, acct, rt.GoodVersion, app), |
| 78 | Digest: rt.GoodDigest, Demand: rt.GoodDemand} |
| 79 | recovery := activator.WakePlan{Boot: good, Mode: RecoveryMode(in.Policy, good.Demand)} |
| 80 | if in.Book.HeldDown(acct, app, installed, host) { |
| 81 | return activator.WakePlan{Refuse: HeldDownRefusal(installed)}, nil |
| 82 | } |
| 83 | if mark, ok := holdRanFor(in.Book, acct, app, host); ok { |
| 84 | return decideHoldRan(in, mark, hasGood) |
| 85 | } |
| 86 | if hasGood && installed == rt.GoodVersion { |
| 87 | return recovery, nil |
| 88 | } |
| 89 | // A record holds the installed build for this instance: boot the good |
| 90 | // build instead. With no good build (every instance, until it first |
| 91 | // serves after the upgrade) there is nothing to fall back to, which is |
| 92 | // today's behaviour and not a regression (spec §8.5, §8.2): the wake |
| 93 | // plans the installed build as a first wake with no fallback, as it |
| 94 | // would had no record been written. Refusing it would turn one |
| 95 | // sibling's failure into an outage for every sleeping sibling. A |
| 96 | // hold-down was refused above and stays refused (spec §6.4). |
| 97 | if hasGood && in.Book.Blocks(acct, app, installed, host, false) { // rolloutsim:M5 |
| 98 | return recovery, nil |
| 99 | } |
| 100 | path := release.VersionPath(in.OptDir, acct, installed, app) |
| 101 | demand, digest, err := in.Demand(installed, path) |
| 102 | if err != nil { |
| 103 | if hasGood { |
| 104 | return recovery, err |
| 105 | } |
| 106 | return activator.WakePlan{Refuse: fmt.Sprintf("build %s is not staged correctly on this box", installed)}, err |
| 107 | } |
| 108 | mode := AttemptMode(in.Policy, demand) |
| 109 | plan := activator.WakePlan{Boot: release.StagedIdentity{Version: installed, StagedPath: path, Digest: digest, Demand: demand}, |
| 110 | Mode: mode, FirstWake: true} |
| 111 | if hasGood && mode == activator.ModeOverlap { |
| 112 | plan.Fallback = &good // rolloutsim:M6 |
| 113 | } |
| 114 | return plan, nil |
| 115 | } |
| 116 | |
| 117 | // decideHoldRan is decideWake for an instance with a hold-ran marker (final |
| 118 | // review wave 2, N2; spec §6.4: once a hold build has executed against an |
| 119 | // instance's database, nothing falls back automatically on that instance). |
| 120 | // The good build never boots in place of the marked build, as a recovery |
| 121 | // or as a fallback, and the marker never lifts a failure record. |
| 122 | // - the installed build is the marked build: it boots, in hold mode with |
| 123 | // no fallback, unless a failure record holds it for the instance (Astra |
| 124 | // R3 N7: spec §8.4 exempts an instance that passed on the build, not one |
| 125 | // where it merely ran). That holds even when the marked build is also |
| 126 | // the good build (Task 3 fix round 1): the instance has served it, so a |
| 127 | // sibling's record does not hold it (Blocks with served), and booting |
| 128 | // it is the marked-build rule, not a fallback. |
| 129 | // |
| 130 | // With a good build, for any other installed build: |
| 131 | // - the installed build is the good build: refused. An installed good |
| 132 | // build is no proof of an explicit rollback, because root's automatic |
| 133 | // sibling repoint installs it too (ruling 205, Astra R3). The proof is |
| 134 | // a signed pointer naming the good build, newer than the marker, and the |
| 135 | // pointer pass (applyPointers) clears the marker when it sees one; the |
| 136 | // next wake then boots the good build as an ordinary wake. The planner |
| 137 | // itself reads no pointer. |
| 138 | // - the installed build is held for the instance by a failure record: |
| 139 | // refused. With the good build barred as well, the instance stays down |
| 140 | // until a retry or a newer build. |
| 141 | // |
| 142 | // Otherwise the installed build boots as a first wake with no fallback: the |
| 143 | // marked build in hold mode, whose failure here is a hold-down, any other |
| 144 | // build in its own mode. With no good build a record alone refuses nothing |
| 145 | // (spec §8.5), so the marked build boots even when a record blocks it. |
| 146 | // |
| 147 | // The hold-down refusal above this call still comes first. The refusal |
| 148 | // words are HeldDownRefusal's for the marked build: no new member text. |
| 149 | func decideHoldRan(in wakeInputs, mark HoldRan, hasGood bool) (activator.WakePlan, error) { |
| 150 | rt := in.Route |
| 151 | acct, app, host := release.RouteAccount(rt), rt.App, rt.Host |
| 152 | installed := rt.Version |
| 153 | switch { |
| 154 | case installed == mark.Build: |
| 155 | if hasGood && in.Book.Blocks(acct, app, installed, host, installed == rt.GoodVersion) { // rolloutsim:M7 |
| 156 | return activator.WakePlan{Refuse: HeldDownRefusal(mark.Build)}, nil |
| 157 | } |
| 158 | case hasGood && installed == rt.GoodVersion: |
| 159 | return activator.WakePlan{Refuse: HeldDownRefusal(mark.Build)}, nil |
| 160 | case hasGood && in.Book.Blocks(acct, app, installed, host, false): |
| 161 | return activator.WakePlan{Refuse: HeldDownRefusal(mark.Build)}, nil |
| 162 | } |
| 163 | path := release.VersionPath(in.OptDir, acct, installed, app) |
| 164 | demand, digest, err := in.Demand(installed, path) |
| 165 | if err != nil { |
| 166 | // A staging refusal cannot boot the good build instead here, as it |
| 167 | // does for an instance with no marker: that would be the fallback. |
| 168 | return activator.WakePlan{Refuse: fmt.Sprintf("build %s is not staged correctly on this box", installed)}, err |
| 169 | } |
| 170 | mode := AttemptMode(in.Policy, demand) |
| 171 | if installed == mark.Build { |
| 172 | mode = activator.ModeHold |
| 173 | } |
| 174 | return activator.WakePlan{Boot: release.StagedIdentity{Version: installed, StagedPath: path, Digest: digest, Demand: demand}, |
| 175 | Mode: mode, FirstWake: true}, nil |
| 176 | } |
| 177 | |
| 178 | // Plan is the activator's Planner (spec §8.2). It is called without |
| 179 | // activator.Service.mu held, before the wake's lease. It also snapshots the |
| 180 | // instance's delivered config revision into the plan (WakePlan.ConfigRev). |
| 181 | // The wake's spawn reads the config file after this moment, so activating |
| 182 | // that revision when the wake is healthy (OnGen, from the revision its |
| 183 | // event carries) can only under-claim, which costs one more config attempt, |
| 184 | // never a missed one. |
| 185 | // |
| 186 | // Every plan gets a token of its own (WakePlan.Token, final review F4), |
| 187 | // taken before anything is registered for it. The activator carries it |
| 188 | // onto every event of the wake, and OnGen acts on a host's wake state |
| 189 | // (Manager.planned, observing, the config activation) only for the token |
| 190 | // that state belongs to, so a late event of an older wake never touches a |
| 191 | // newer one's. |
| 192 | // |
| 193 | // A Refuse text reaches the member's "wake failed" event (the activator's |
| 194 | // wake error), so it names builds only, never an error's text. |
| 195 | // |
| 196 | // Every plan for a managed host, refused or not, replaces the host's entry |
| 197 | // in Manager.planned, so prune keeps the failure record of a first wake's |
| 198 | // build until that wake is over. The installed build is registered there |
| 199 | // before the planner's slow reads (planRow), so a Pass that runs while they |
| 200 | // do keeps a record they could no longer see. |
| 201 | // |
| 202 | // The row is read again after those reads. A Pass whose snapshot preceded |
| 203 | // the registration could have pruned the build's record only if no route |
| 204 | // installed the build when that Pass listed them, so this host's row had |
| 205 | // already moved off it, and the read after the registration sees that. Such |
| 206 | // a plan is made once more against the fresh row. A row that moves again |
| 207 | // during the replan refuses the wake rather than chase the row: two installs |
| 208 | // inside one plan are installs still landing, and the activator's next wake |
| 209 | // plans against whatever settled. |
| 210 | func (m *Manager) Plan(ctx context.Context, rt registry.Route) (plan activator.WakePlan) { |
| 211 | if !Managed(rt) { |
| 212 | return activator.WakePlan{} |
| 213 | } |
| 214 | // Joined by Close for its whole length (spec §8.6, Task 11 fix round 1): |
| 215 | // a plan under way may still write its wake's intent. Deferred first, so |
| 216 | // it is given back last, after beginWake below has counted the wake |
| 217 | // whose intent it wrote: the count never touches zero in between. |
| 218 | m.enter(false) |
| 219 | defer m.leave() |
| 220 | tok, planAt := m.newWakeToken(rt.Host) |
| 221 | defer func() { |
| 222 | plan.Token = tok |
| 223 | // Write-ahead (Task 5, Astra R1 #5): a first wake's intent is on disk |
| 224 | // before the activator takes the lease, with no activator lock held |
| 225 | // here. It ends on the wake's terminal event (OnGen). A plan whose |
| 226 | // intent could not be written boots nothing, with the approved |
| 227 | // refusal for a failure book this box cannot keep, and is |
| 228 | // registered as the refusal it now is. |
| 229 | plan = m.beginWake(rt, plan, tok, planAt) |
| 230 | m.notePlanned(rt, plan, tok) |
| 231 | // The book's snapshot is taken here, at the plan's end, after every |
| 232 | // book read the planner made (decideWake, the replan): the wake's |
| 233 | // failure is stale only after a retry the plan could not have seen |
| 234 | // (ruling 191, Task 4 fix round 1). A retry landing during the slow |
| 235 | // reads (Materialize, the policy load) let this plan boot the build, |
| 236 | // so the build's failure in it is a real one. Its good write clears |
| 237 | // only markers set before here (Astra R3 N6): the exec follows the |
| 238 | // plan, and the plan's check re-reads the marker there. |
| 239 | m.noteWakeStart(rt.Host, tok, planAt, m.d.Book.Snapshot(), plan) |
| 240 | }() |
| 241 | if m.attemptRunning(rt.Host) { |
| 242 | // The instance's attempt has not reported yet (spec §6.4, §8.1): a |
| 243 | // hold that went down, or a swap whose entry stopped under it, is |
| 244 | // recorded by done() before anything wakes the instance, so no wake |
| 245 | // can boot the good build ahead of a hold-down record. |
| 246 | return activator.WakePlan{Refuse: movingRefusal} |
| 247 | } |
| 248 | if rt.Version == "" { |
| 249 | // A cold managed host (spec §6.4, §8.4): its build is installed and |
| 250 | // stamped first, and the wake is planned against that installed |
| 251 | // identity, with its failure records and its first-wake canary. |
| 252 | // Nothing managed ever runs through an empty plan: the live-link |
| 253 | // wake would materialize the channel head and exec it unplanned. |
| 254 | if m.d.Materialize == nil { |
| 255 | return activator.WakePlan{Refuse: "no build is installed for this instance yet"} |
| 256 | } |
| 257 | if err := m.d.Materialize(ctx, rt.Host); err != nil { |
| 258 | m.d.Logf("rollout: %s: install before its wake: %v", rt.Host, err) |
| 259 | return activator.WakePlan{Refuse: "its build could not be installed"} |
| 260 | } |
| 261 | fresh, err := m.d.Routes.Get(rt.Host) |
| 262 | if err != nil || fresh == nil || fresh.Version == "" { |
| 263 | if err != nil { |
| 264 | m.d.Logf("rollout: %s: read the row after its install: %v", rt.Host, err) |
| 265 | } |
| 266 | return activator.WakePlan{Refuse: "no build is installed for this instance yet"} |
| 267 | } |
| 268 | rt = *fresh |
| 269 | } |
| 270 | for again := false; ; again = true { |
| 271 | plan = m.planRow(ctx, rt, tok) |
| 272 | fresh, err := m.d.Routes.Get(rt.Host) |
| 273 | if err != nil || fresh == nil { |
| 274 | // Nothing confirms the row the plan was made against, so nothing |
| 275 | // may boot from it: the same answer as an unreadable row after a |
| 276 | // cold install. |
| 277 | if err != nil { |
| 278 | m.d.Logf("rollout: %s: read the row after its plan: %v", rt.Host, err) |
| 279 | } |
| 280 | return activator.WakePlan{Refuse: "no build is installed for this instance yet"} |
| 281 | } |
| 282 | if fresh.Version == rt.Version { |
| 283 | return plan |
| 284 | } |
| 285 | if again { |
| 286 | m.d.Logf("rollout: %s: its installed build moved twice while its wake was planned (%s, now %s); refusing the wake", rt.Host, rt.Version, fresh.Version) |
| 287 | return activator.WakePlan{Refuse: "no build is installed for this instance yet"} |
| 288 | } |
| 289 | m.d.Logf("rollout: %s: its installed build moved from %s to %s while its wake was planned; planning again", rt.Host, rt.Version, fresh.Version) |
| 290 | rt = *fresh |
| 291 | } |
| 292 | } |
| 293 | |
| 294 | // movingRefusal is a wake's refusal while the instance's attempt has not |
| 295 | // reported (Plan's attemptRunning check, and beginWake meeting the |
| 296 | // attempt's intent between its slot freeing and its End). |
| 297 | const movingRefusal = "this instance is still moving to another build" |
| 298 | |
| 299 | // beginWake writes the intent of plan, a first wake planned with token tok |
| 300 | // at planAt (Task 5 rule 1), and returns the plan to hand the activator: |
| 301 | // plan itself, or a refusal when the intent could not be written. Any |
| 302 | // other plan writes nothing. The intent's mode is the plan's: the activator |
| 303 | // runs a planned wake in the mode it was planned in (a foreign-layout row |
| 304 | // gets one generation at a time, so no override applies; its events carry |
| 305 | // plan.Mode), and that is the mode a failure of the wake records. |
| 306 | func (m *Manager) beginWake(rt registry.Route, plan activator.WakePlan, tok uint64, planAt time.Time) activator.WakePlan { |
| 307 | if !plan.FirstWake || plan.Boot.Version == "" || plan.Refuse != "" { |
| 308 | return plan |
| 309 | } |
| 310 | in := Intent{Account: release.RouteAccount(rt), App: rt.App, Build: plan.Boot.Version, Mode: string(plan.Mode), |
| 311 | ID: m.wakeExecID(tok), Start: planAt} |
| 312 | err := m.d.Book.Begin(rt.Host, in) |
| 313 | if err == nil { |
| 314 | // The intent is on disk: Close now waits for the terminal event |
| 315 | // that ends it (OnGen, leaveWake). A refused plan, and a Begin that |
| 316 | // failed, wrote nothing and is never counted. |
| 317 | m.enterWake(tok) |
| 318 | return plan |
| 319 | } |
| 320 | m.d.Logf("rollout: %s: write the intent of its first wake of %s: %v; refusing the wake", rt.Host, plan.Boot.Version, err) |
| 321 | if errors.Is(err, ErrIntentStands) { |
| 322 | // The instance's attempt finished and has not yet ended its intent: |
| 323 | // it is still, for a moment, moving. Not a book fault. |
| 324 | return activator.WakePlan{Refuse: movingRefusal, Token: tok} |
| 325 | } |
| 326 | return activator.WakePlan{Refuse: BookUnreadableRefusal, Token: tok} |
| 327 | } |
| 328 | |
| 329 | // planRow is one plan against rt (Plan's body, after the cold install), for |
| 330 | // the plan whose token is tok. It registers rt's installed build as the |
| 331 | // host's in-flight planned build before any slow read (the policy, the |
| 332 | // book's reading in decideWake, the canonical demand, the failing pointer); |
| 333 | // Plan's deferred notePlanned replaces or clears that entry with what the |
| 334 | // plan finally boots. |
| 335 | func (m *Manager) planRow(ctx context.Context, rt registry.Route, tok uint64) activator.WakePlan { |
| 336 | m.notePlanning(rt, tok) |
| 337 | acct, app := release.RouteAccount(rt), rt.App |
| 338 | in := wakeInputs{Route: rt, OptDir: m.d.OptDir, Policy: m.policies.get(ctx, acct, app), Book: m.d.Book, |
| 339 | Demand: func(ver, path string) (string, string, error) { |
| 340 | // The same reading an adoption attempt gets (resolve): only a |
| 341 | // staging fault is an error, and anything unverifiable is hold. |
| 342 | return m.stagedDemand(ctx, rt, ver, path) |
| 343 | }} |
| 344 | // The delivered revision now, before the wake's spawn reads the file. |
| 345 | // It travels with the plan and comes back on the wake's events, so no |
| 346 | // later plan for the host can change what this wake activates. |
| 347 | m.mu.Lock() |
| 348 | rev := m.track(rt.Host).delivered |
| 349 | m.mu.Unlock() |
| 350 | plan, err := decideWake(in) |
| 351 | plan.ConfigRev = rev |
| 352 | if err != nil { |
| 353 | m.stagingRefused(rt, rt.Version, err) |
| 354 | } |
| 355 | if plan.FirstWake && plan.Boot.Version != "" { |
| 356 | // Read now, before the wake runs, so a failure of the build is |
| 357 | // published with no blocking read (buildFailed, spec §8.1). |
| 358 | plan.FailedUnder = m.failedUnder(ctx, rt, plan.Boot.Version) |
| 359 | } |
| 360 | if plan.Boot.StagedPath != "" { |
| 361 | plan.Check = m.planCheck(rt, plan) |
| 362 | } |
| 363 | return plan |
| 364 | } |
| 365 | |
| 366 | // planCheck is a plan's selection gate (WakePlan.Check), asked by the |
| 367 | // activator at each planned exec, after the lease and the restore (spec |
| 368 | // §7.4, §8.1–8.4): a first wake's build that a failure record now holds for |
| 369 | // this instance (a sibling's failure since the plan, or a hold-down) may not |
| 370 | // boot. The good build always may: it is what a held build falls back to. |
| 371 | // An instance with no good build is refused only by a hold-down: a record |
| 372 | // alone would not move it onto anything, and it has no fallback by design |
| 373 | // until it first serves (spec §8.5), so refusing it would be an outage. |
| 374 | // |
| 375 | // A hold-ran marker (final review wave 2, N2), set since the plan or |
| 376 | // before it, applies decideHoldRan's rule at the exec: |
| 377 | // - the marked build may not boot while it is held down there or, with a |
| 378 | // good build, while a failure record holds it for the instance (Astra |
| 379 | // R3 N7: the marker lifts no record). When it is also the good build it |
| 380 | // is under this rule only as the installed build (Task 3 fix round 1), |
| 381 | // and a record does not hold a build the instance served; |
| 382 | // - otherwise the good build may not boot, whatever the route installs: |
| 383 | // as a recovery, a fallback, an untried first wake's stand-in, or the |
| 384 | // installed build itself (ruling 205: only the pointer pass's clearing |
| 385 | // of the marker on a signed rollback lets it boot again, and the check |
| 386 | // reads no pointer); |
| 387 | // - any other build is checked as without a marker. |
| 388 | // |
| 389 | // The book is in memory; nothing here blocks. |
| 390 | func (m *Manager) planCheck(rt registry.Route, plan activator.WakePlan) func(build string) error { |
| 391 | acct, app, host := release.RouteAccount(rt), rt.App, rt.Host |
| 392 | boot := plan.Boot.Version |
| 393 | hasGood := rt.GoodVersion != "" && rt.GoodDigest != "" |
| 394 | // The texts reach the member's "wake failed" event (memberSafeWakeReason |
| 395 | // passes a planned wake's error through). |
| 396 | held := func(build string) error { |
| 397 | return fmt.Errorf("build %s failed on this box while this instance was starting, so it was not started", build) |
| 398 | } |
| 399 | return func(build string) error { |
| 400 | if mark, ok := holdRanFor(m.d.Book, acct, app, host); ok { |
| 401 | isGood := hasGood && build == rt.GoodVersion |
| 402 | switch { |
| 403 | case build == mark.Build && (!isGood || rt.Version == mark.Build): |
| 404 | if m.d.Book.HeldDown(acct, app, build, host) || (hasGood && m.d.Book.Blocks(acct, app, build, host, isGood)) { |
| 405 | return held(build) |
| 406 | } |
| 407 | return nil |
| 408 | case isGood: |
| 409 | return errors.New(HeldDownRefusal(mark.Build)) |
| 410 | } |
| 411 | } |
| 412 | if !plan.FirstWake || build != boot { |
| 413 | return nil |
| 414 | } |
| 415 | if m.d.Book.HeldDown(acct, app, build, host) || (hasGood && m.d.Book.Blocks(acct, app, build, host, false)) { |
| 416 | return held(build) |
| 417 | } |
| 418 | return nil |
| 419 | } |
| 420 | } |
| 421 | |
| 422 | // attemptRunning reports whether host has an attempt in flight: admitted |
| 423 | // and not yet finished by done(). |
| 424 | func (m *Manager) attemptRunning(host string) bool { |
| 425 | s := m.schedulerOf(host) |
| 426 | if s == nil { |
| 427 | return false |
| 428 | } |
| 429 | s.mu.Lock() |
| 430 | defer s.mu.Unlock() |
| 431 | h := s.q.hosts[host] |
| 432 | return h != nil && h.running != nil |
| 433 | } |
| 434 | |
| 435 | // OnGen is the activator's OnGen: a planned wake's outcome (spec §6.4, §8.2, |
| 436 | // §8.5). A healthy wake records the serving build and activates the config |
| 437 | // it started with; a first wake becomes the good build only once it has |
| 438 | // survived its 10 s; a failed one goes through the cohort-failure |
| 439 | // transition, held down when it was a hold build. It is never called with |
| 440 | // activator.Service.mu held (emitGen), which served and publishServing need: |
| 441 | // they take goodMu and call back into Current. |
| 442 | // |
| 443 | // The host's wake state is the event's only when the token matches (final |
| 444 | // review F4): the planned entry and the observation each end only by an |
| 445 | // event carrying their own token, and only the host's newest wake (the |
| 446 | // token of its last plan) opens an observation or activates a config |
| 447 | // revision, the one its event carries. A late event of an older wake (an |
| 448 | // observer's delayed GenCancelled, say) leaves a newer wake's registration, |
| 449 | // observation and config alone. What an event says about its build is |
| 450 | // still acted on whoever's it is: a failure is recorded, and a serving or |
| 451 | // good build is published only while its generation is the one the |
| 452 | // activator serves (acknowledged). A token of zero matches nothing. |
| 453 | func (m *Manager) OnGen(ev activator.GenEvent) { |
| 454 | // Joined by Close (spec §8.6): a wake's terminal event writes its outcome |
| 455 | // and ends its intent, which must be on disk before the agent exits. |
| 456 | m.enter(false) |
| 457 | defer m.leave() |
| 458 | rt := ev.Route |
| 459 | // The wake's start, its plan: a hold-ran marker it leaves carries the |
| 460 | // plan time (ruling 205), and its good write and its failure are ordered |
| 461 | // against the book by the plan's snapshot (Astra R3 N6, ruling 191). |
| 462 | // Every kind but a first wake's healthy is the wake's last event, which |
| 463 | // lets the entry go: an ordinary wake (an untried first wake's good |
| 464 | // build included) reports healthy once and nothing after. |
| 465 | ended := ev.Kind != activator.GenHealthy || !ev.FirstWake |
| 466 | started := m.wakeStarted(rt.Host, ev.Token, ended) |
| 467 | m.mu.Lock() |
| 468 | newest := ev.Token != 0 && m.wakeTok[rt.Host] == ev.Token |
| 469 | // The wake has an outcome: its build is no longer only planned, so its |
| 470 | // failure record need not outlive the install for its sake. |
| 471 | if w, ok := m.planned[rt.Host]; ok && ev.Token != 0 && w.token == ev.Token { |
| 472 | delete(m.planned, rt.Host) |
| 473 | } |
| 474 | switch { |
| 475 | case ev.Kind == activator.GenHealthy && ev.FirstWake: |
| 476 | if newest { |
| 477 | m.observing[rt.Host] = ev.Token |
| 478 | } |
| 479 | case ev.Kind != activator.GenHealthy: |
| 480 | // GenSettled, GenFailed, GenFellBack and GenCancelled (an |
| 481 | // intentional stop ended the observation with no verdict) all close |
| 482 | // the observation their wake opened. The switch below has no |
| 483 | // recording arm for GenCancelled, so a cancelled observation records |
| 484 | // nothing (spec §6.4). |
| 485 | if tok, ok := m.observing[rt.Host]; ok && ev.Token != 0 && tok == ev.Token { |
| 486 | delete(m.observing, rt.Host) |
| 487 | } |
| 488 | } |
| 489 | m.mu.Unlock() |
| 490 | defer m.changed() // after the event's own work, with no lock held |
| 491 | switch ev.Kind { |
| 492 | case activator.GenHealthy: |
| 493 | // A wake that is not a first wake includes the good build booted |
| 494 | // untried under Task 9's verdict floor: an ordinary wake of the good |
| 495 | // build, recorded as served, with nothing against the build it skipped. |
| 496 | if !ev.FirstWake { |
| 497 | m.served(rt.Host, ev.Gen, started.snap) |
| 498 | } |
| 499 | // A first wake is published when it settles: until then the |
| 500 | // activator hides it (Current), so it is not yet acknowledged. |
| 501 | m.wakeConfigActivated(rt.Host, ev.Token, ev.ConfigRev) |
| 502 | case activator.GenSettled: |
| 503 | m.served(rt.Host, ev.Gen, started.snap) |
| 504 | case activator.GenFellBack: |
| 505 | // The failure hold first (spec §8.1): nothing that reads the |
| 506 | // serving build or the active config may see the fallback before the |
| 507 | // failed build is held. |
| 508 | m.buildFailed(rt, ev, started) |
| 509 | m.publishServing(rt.Host, ev.Gen) |
| 510 | m.wakeConfigActivated(rt.Host, ev.Token, ev.ConfigRev) |
| 511 | m.emit(rt, EvFallbackServed, "warn", textFallback(ev.Failed.Version, ev.Gen.Build)) |
| 512 | case activator.GenFailed: |
| 513 | m.buildFailed(rt, ev, started) |
| 514 | case activator.GenCancelled: |
| 515 | // An intentional stop ended the first wake's observation, or the |
| 516 | // wake's lifecycle ended it after its build ran (spec §6.4): no |
| 517 | // verdict on the build, so nothing is recorded or published. A hold |
| 518 | // build that ran leaves the instance's hold-ran marker (final review |
| 519 | // wave 2, N2): its database may be the build's now, so the next wake |
| 520 | // must not boot the good build. A fact about the instance, like a |
| 521 | // failure, so it is not token-scoped. |
| 522 | if ev.Executed && ev.FirstWake && ev.Mode == activator.ModeHold && ev.Gen.Build != "" { |
| 523 | // At is the wake's plan time, its execution's start (ruling 205): |
| 524 | // a rollback signed while the wake ran is newer than the marker. |
| 525 | if started.at.IsZero() { |
| 526 | m.d.Logf("rollout: %s: the start of the wake that ran %s is not known; its hold-ran marker is stamped now", rt.Host, ev.Gen.Build) |
| 527 | } |
| 528 | mark := HoldRan{Account: release.RouteAccount(rt), App: rt.App, Build: ev.Gen.Build, Exec: m.wakeExecID(ev.Token), At: started.at} |
| 529 | if err := m.d.Book.MarkHoldRan(rt.Host, mark); err != nil { |
| 530 | m.d.Logf("rollout: %s: write hold-ran marker for %s: %v (held in memory)", rt.Host, mark.Build, err) |
| 531 | } |
| 532 | m.pointerReadDue(appKey{mark.Account, mark.App}) |
| 533 | } |
| 534 | } |
| 535 | // The wake's intent ends with the first event that settles its planned |
| 536 | // build's fate (Task 5 rule 2, plan review #1), after that event's own |
| 537 | // book write above (buildFailed's record, the hold-ran marker): settled, |
| 538 | // failed, fell back, cancelled (with or without a run), and healthy when |
| 539 | // it is not a first wake's, which is the good build booted under the |
| 540 | // verdict floor with the planned build never run. A first wake's healthy |
| 541 | // event opens its observation and is not the end. An older wake's event |
| 542 | // carries its own token, so it never ends a newer wake's intent. Not |
| 543 | // under activator.Service.mu (emitGen's rule). |
| 544 | if ended { |
| 545 | if err := m.d.Book.End(rt.Host, m.wakeExecID(ev.Token)); err != nil { |
| 546 | m.d.Logf("rollout: %s: end the intent of its wake: %v (it stays on disk until a save lands)", rt.Host, err) |
| 547 | } |
| 548 | // The wake's count in Close's join goes with the End (beginWake |
| 549 | // took it). This call is itself counted until it returns, so the |
| 550 | // join cannot see zero before the End above is written. A wake |
| 551 | // replaced by a newer plan has its intent replaced on disk, but its |
| 552 | // own terminal event still ends its count here: the activator sends |
| 553 | // exactly one per planned wake (afterWake, cancelPlanned). |
| 554 | m.leaveWake(ev.Token) |
| 555 | } |
| 556 | } |
| 557 | |
| 558 | // wakeConfigActivated is configActivated for a planned wake's event (final |
| 559 | // review wave 2, N3): the revision activates only while tok is still the |
| 560 | // host's newest wake token, and that check and the activation are one |
| 561 | // critical section of m.mu. A check made earlier (OnGen's newest) can go |
| 562 | // stale while the event does its other work (served's registry write): a |
| 563 | // newer wake could plan and activate its own revision meanwhile, and this |
| 564 | // older one would then put its revision back. The held ack is written, and |
| 565 | // the queue's pending config attempt cleared, after the lock is released. |
| 566 | func (m *Manager) wakeConfigActivated(host string, tok uint64, rev string) { |
| 567 | if rev == "" || tok == 0 { |
| 568 | return |
| 569 | } |
| 570 | m.mu.Lock() |
| 571 | if m.wakeTok[host] != tok { |
| 572 | m.mu.Unlock() |
| 573 | return |
| 574 | } |
| 575 | write, _ := m.track(host).activate(rev) |
| 576 | ack := m.ackLocked(write) |
| 577 | m.mu.Unlock() |
| 578 | m.activationDone(host, rev, ack) |
| 579 | } |
| 580 | |
| 581 | // buildFailed is a failed first wake's cohort-failure transition (spec |
| 582 | // §8.1, §8.2 step 4). started is the wake's start (wakeStarted): a wake |
| 583 | // planned before a retry of its build fails stale (ruling 191), and runs |
| 584 | // none of the transition. |
| 585 | func (m *Manager) buildFailed(rt registry.Route, ev activator.GenEvent, started wakeStart) { |
| 586 | // No blocking read before the record (spec §8.1): the failing pointer's |
| 587 | // PromotedAt was read when the wake was planned (WakePlan.FailedUnder). |
| 588 | rec := FailureRecord{Account: release.RouteAccount(rt), App: rt.App, Build: ev.Failed.Version, Host: rt.Host, |
| 589 | Cause: string(activator.CauseRelease), Mode: string(ev.Mode), Reason: ev.Reason, |
| 590 | PromotedAt: ev.FailedUnder, At: m.d.Now(), |
| 591 | // The wake's start for a stale failure's marker: Record sets one in |
| 592 | // place of a hold-down when the failure is pre-retry (ruling 191). |
| 593 | Seq: started.snap, HoldRanAt: started.at, HoldRanExec: m.wakeExecID(ev.Token)} |
| 594 | if rec.PromotedAt.IsZero() { |
| 595 | // No pointer named the build when the wake was planned: the |
| 596 | // failure's own time, so only a pointer signed after it clears the |
| 597 | // record (as done() does for an attempt). |
| 598 | rec.PromotedAt = rec.At |
| 599 | } |
| 600 | if ev.Mode == activator.ModeHold { |
| 601 | // A hold build executed against this instance's database: nothing |
| 602 | // falls back on it automatically (spec §6.4, §8.3). |
| 603 | rec.HeldDown = []string{rt.Host} |
| 604 | } |
| 605 | s := m.schedulerOf(rt.Host) |
| 606 | if s == nil { |
| 607 | // No scheduler for the host's app: the record still halts the |
| 608 | // cohort, and the halt is still announced, on the same condition |
| 609 | // cohortFailed announces it (a newly recorded failure, never a |
| 610 | // pre-retry one or one merged into a record that stands). Without |
| 611 | // it a member whose first wake failed here saw the fallback event |
| 612 | // and never the halt (Astra R3 flag triage T11). |
| 613 | res, err := m.d.Book.Record(rec) |
| 614 | if err != nil { |
| 615 | m.d.Logf("rollout: write failure record %s: %v (held in memory)", rec.Build, err) |
| 616 | } |
| 617 | if res == Recorded { |
| 618 | m.d.Logf("rollout: %s/%s build %s failed on %s: %s", rec.Account, rec.App, rec.Build, rec.Host, rec.Reason) |
| 619 | m.emit(rt, EvCohortHalted, "warn", textCohortHalted(rec.Build, m.d.Box, reasonWord(rec.Reason))) |
| 620 | } |
| 621 | return |
| 622 | } |
| 623 | m.cohortFailed(s, rt, rec) |
| 624 | } |
| 625 | |
amadan