| 1 | // wakeplan.go is the activator's half of the rollout scheduler (zero-downtime |
| 2 | // deploys spec §6.4, §8.2): which build a wake boots, the first-wake budget, |
| 3 | // the fallback to the good build, and the 10 s observation. All of it is |
| 4 | // reached through Service.Planner, which only CARLOS_SWAP=overlap sets. With |
| 5 | // Planner nil (the flag off) nothing here runs. |
| 6 | package activator |
| 7 | |
| 8 | import ( |
| 9 | "context" |
| 10 | "errors" |
| 11 | "fmt" |
| 12 | "log" |
| 13 | "sync" |
| 14 | "time" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/registry" |
| 17 | "github.com/carlosframework/platform/internal/release" |
| 18 | "github.com/carlosframework/platform/internal/rolloutwords" |
| 19 | ) |
| 20 | |
| 21 | // WakePlan is the scheduler's answer to "which build does this wake boot" |
| 22 | // (spec §8.2). The zero plan keeps today's wake. |
| 23 | type WakePlan struct { |
| 24 | // Boot is the artifact to exec, by its immutable staged path. |
| 25 | Boot release.StagedIdentity |
| 26 | Mode SwapMode |
| 27 | // FirstWake: Boot has never served on this instance. The wake is a canary |
| 28 | // for it, with the 55 s budget and the 10 s observation. |
| 29 | FirstWake bool |
| 30 | // Fallback is the good build, booted when a first wake in overlap mode |
| 31 | // fails. Nil when there is none. |
| 32 | Fallback *release.StagedIdentity |
| 33 | // Refuse, when set, fails the wake before the lease with this reason. |
| 34 | Refuse string |
| 35 | // FailedUnder is the PromotedAt of the pointer Boot was promoted under, |
| 36 | // read by the planner before the wake runs, so a failed first wake is |
| 37 | // published with no blocking read (spec §8.1), as dispatch.FailedUnder |
| 38 | // is for an attempt. Zero: the failure's own time stands in. |
| 39 | FailedUnder time.Time |
| 40 | // Check is the planner's selection gate (spec §7.4, §8.1–8.4): nil when |
| 41 | // build may still boot. Planning runs before the lease and the restore, |
| 42 | // so a sibling's failure recorded since can hold the build it picked. |
| 43 | // The activator asks it at every planned exec, after the restore; the |
| 44 | // good build passes unless a hold build has run on the instance since it |
| 45 | // last passed (the scheduler's hold-ran marker), where nothing falls |
| 46 | // back. Nil: nothing to re-check. It is called without a.mu held. |
| 47 | Check func(build string) error |
| 48 | // Token identifies this planned wake (final review F4). The planner |
| 49 | // assigns it, unique per plan it makes (a replan, checkPlan's, is a new |
| 50 | // plan with a new token), before it registers anything for the wake. |
| 51 | // The activator never makes or changes one: it copies the token of the |
| 52 | // plan the wake ran, unchanged, into every GenEvent of that wake, |
| 53 | // afterWake's and the first-wake observer's, so the scheduler can act |
| 54 | // only on the wake it registered. Zero: the planner matches nothing by |
| 55 | // token. |
| 56 | Token uint64 |
| 57 | // ConfigRev is the config revision the planner snapshotted for this |
| 58 | // wake: the delivered revision to activate once the wake serves. The |
| 59 | // activator does not know which revision a wake started with (it reads |
| 60 | // the instance's env file at the spawn), so it carries this one, |
| 61 | // unchanged, into every GenEvent of the wake, beside Token. |
| 62 | ConfigRev string |
| 63 | } |
| 64 | |
| 65 | func (p WakePlan) planned() bool { return p.Boot.StagedPath != "" } |
| 66 | |
| 67 | // errPlanStale: the planned build is no longer the one to boot. A |
| 68 | // supersession (gateSuperseded), so it records nothing and falls back to |
| 69 | // nothing (classifyStep's stepStale). |
| 70 | var errPlanStale = fmt.Errorf("the planned build is no longer the one to boot: %w", errSwapSuperseded) |
| 71 | |
| 72 | // checkPlan is a planned wake's selection gate before its spawn (spec §7.4, |
| 73 | // §8.1–8.4), after the lease and the restore: a refused plan is planned |
| 74 | // again once, on the state already restored, and a second refusal abandons |
| 75 | // the wake (errPlanStale). It returns the plan to boot. The exec itself asks |
| 76 | // the returned plan's Check once more (planGate), with no second replan. |
| 77 | // The planner and Check are called without a.mu held. |
| 78 | func (a *Service) checkPlan(ctx context.Context, e *actEntry, rt registry.Route, plan WakePlan) (WakePlan, error) { |
| 79 | if plan.Check == nil { |
| 80 | return plan, nil |
| 81 | } |
| 82 | err := plan.Check(plan.Boot.Version) |
| 83 | if err == nil { |
| 84 | return plan, nil |
| 85 | } |
| 86 | if a.Planner == nil { |
| 87 | return plan, fmt.Errorf("%w: %v", errPlanStale, err) |
| 88 | } |
| 89 | next := a.Planner(ctx, rt) |
| 90 | if next.Refuse != "" || !next.planned() { |
| 91 | return plan, fmt.Errorf("%w: %v; replanned: %s", errPlanStale, err, next.Refuse) |
| 92 | } |
| 93 | if next.Check != nil { |
| 94 | if nerr := next.Check(next.Boot.Version); nerr != nil { |
| 95 | // The replan never becomes the wake's plan, so afterWake will |
| 96 | // report only the first plan: the replan's own terminal event |
| 97 | // is sent here (rollout Task 5). Nothing of it ran. A planner |
| 98 | // that handed out the same token again gets one event for it. |
| 99 | if next.Token != plan.Token { |
| 100 | a.cancelPlanned(rt, next, false) |
| 101 | } |
| 102 | return plan, fmt.Errorf("%w: %v", errPlanStale, nerr) |
| 103 | } |
| 104 | } |
| 105 | a.mu.Lock() |
| 106 | e.plan, e.firstWake = next, next.FirstWake |
| 107 | a.mu.Unlock() |
| 108 | // The replaced plan's build never ran (its check refused it before the |
| 109 | // exec), and afterWake now reports the replan: the replaced plan's |
| 110 | // terminal event is sent here (rollout Task 5). |
| 111 | if plan.Token != next.Token { |
| 112 | a.cancelPlanned(rt, plan, false) |
| 113 | } |
| 114 | // The replanned build (and its fallback) is published before it execs |
| 115 | // (spec §4.2, §8.5). A failed write execs neither: the wake fails with |
| 116 | // errStagedBuildNotPublished, which is abandonment, never a failure. |
| 117 | if err := a.publishBeforeExec(); err != nil { |
| 118 | return next, err |
| 119 | } |
| 120 | return next, nil |
| 121 | } |
| 122 | |
| 123 | // wakeOwned is the ownership check at a wake's exec (spec §7.4): the wake's |
| 124 | // lifecycle has not ended, and plan 3's ownedLocked holds for a waking |
| 125 | // entry (the entry is the host's, no fence, no shutdown; its lease is the |
| 126 | // wake's own, which planGate checks beside it). |
| 127 | func (a *Service) wakeOwned(ctx context.Context, e *actEntry) error { |
| 128 | if err := ctx.Err(); err != nil { |
| 129 | return err |
| 130 | } |
| 131 | a.mu.Lock() |
| 132 | defer a.mu.Unlock() |
| 133 | return a.ownedLocked(e, true) |
| 134 | } |
| 135 | |
| 136 | // planGate is the gate every planned exec passes, B's and its fallback's |
| 137 | // alike, asked immediately before the exec (startPinnedAt's gate, or |
| 138 | // startWorker's before-exec hook): the wake's locally held lease |
| 139 | // (leaseLost; the entry has no replica yet, so ownedLocked cannot see it), |
| 140 | // the ownership check (wakeOwned), and the plan's selection gate for the |
| 141 | // build about to exec. Every refusal is abandonment, never a failure of the |
| 142 | // build: the lease and ownership refusals are gateNoOwnership, the |
| 143 | // selection gate's is errPlanStale (gateSuperseded), a cancelled wake is |
| 144 | // the lifecycle's. Check runs after a.mu is released. |
| 145 | func (a *Service) planGate(ctx context.Context, e *actEntry, lease Lease, plan WakePlan, build string) func() error { |
| 146 | return func() error { |
| 147 | if err := leaseLost(lease); err != nil { |
| 148 | return err |
| 149 | } |
| 150 | if err := a.wakeOwned(ctx, e); err != nil { |
| 151 | return err |
| 152 | } |
| 153 | if plan.Check != nil { |
| 154 | if err := plan.Check(build); err != nil { |
| 155 | return fmt.Errorf("%w: %v", errPlanStale, err) |
| 156 | } |
| 157 | } |
| 158 | return nil |
| 159 | } |
| 160 | } |
| 161 | |
| 162 | func genOf(id release.StagedIdentity) Gen { |
| 163 | return Gen{Build: id.Version, StagedPath: id.StagedPath, Digest: id.Digest, Demand: id.Demand} |
| 164 | } |
| 165 | |
| 166 | // workerGenOf is a worker's generation as the scheduler sees it: its |
| 167 | // captured staged artifact, marked Worker, with no number. The scheduler |
| 168 | // compares a worker's generations by their artifact (rollout's |
| 169 | // acknowledged), because a respawn on the same entry changes the build |
| 170 | // without a number of its own. |
| 171 | func workerGenOf(id release.StagedIdentity) Gen { |
| 172 | g := genOf(id) |
| 173 | g.Worker = true |
| 174 | return g |
| 175 | } |
| 176 | |
| 177 | // withIdentity is g, the generation a planned exec started, carrying the |
| 178 | // staged identity it was booted from (PF10, spec §12 "Legacy sidecar |
| 179 | // binding"): the build, its staged path, digest and demand. The legacy |
| 180 | // sidecar co-spawned with it execs that staged path (startSidecar), and the |
| 181 | // scheduler records it as good by its digest. |
| 182 | func withIdentity(g Gen, id release.StagedIdentity) Gen { |
| 183 | g.Build, g.StagedPath, g.Digest, g.Demand = id.Version, id.StagedPath, id.Digest, id.Demand |
| 184 | return g |
| 185 | } |
| 186 | |
| 187 | // GenEventKind names what happened to a planned wake. |
| 188 | type GenEventKind string |
| 189 | |
| 190 | const ( |
| 191 | GenHealthy GenEventKind = "healthy" // the planned build is serving |
| 192 | GenSettled GenEventKind = "settled" // a first wake survived its 10 s |
| 193 | GenFailed GenEventKind = "failed" // the planned build failed, nothing fell back |
| 194 | GenFellBack GenEventKind = "fellback" // the planned build failed, the good build serves |
| 195 | // GenCancelled: a first wake's observation ended without a verdict (an |
| 196 | // intentional stop, spec §6.4). It records nothing about the build; it |
| 197 | // only closes the observation GenHealthy opened. |
| 198 | GenCancelled GenEventKind = "cancelled" |
| 199 | ) |
| 200 | |
| 201 | // GenEvent is a planned wake's outcome, for the scheduler. |
| 202 | type GenEvent struct { |
| 203 | Kind GenEventKind |
| 204 | Route registry.Route |
| 205 | Gen Gen // the generation serving (healthy, settled, fellback) |
| 206 | Failed release.StagedIdentity // the build that failed (failed, fellback) |
| 207 | Mode SwapMode |
| 208 | FirstWake bool |
| 209 | Reason string |
| 210 | At time.Time |
| 211 | // FailedUnder is the plan's FailedUnder (failed, fellback): the |
| 212 | // failing pointer's PromotedAt, read before the wake ran. |
| 213 | FailedUnder time.Time |
| 214 | // Token and ConfigRev are those of the plan the wake ran |
| 215 | // (WakePlan.Token, WakePlan.ConfigRev; final review F4). Every event of |
| 216 | // one planned wake carries the same pair, on every kind. |
| 217 | Token uint64 |
| 218 | ConfigRev string |
| 219 | // Executed (cancelled only): the first wake's build ran, or may have, |
| 220 | // before the wake ended with no verdict, and Gen.Build names it (final |
| 221 | // review wave 2, N2). Always so for a cancelled observation, which |
| 222 | // follows a healthy wake. A hold first wake that its lifecycle ended |
| 223 | // after the exec (a stop, a fence, a failure after the spawn) is |
| 224 | // reported the same way: the build may have migrated the database, so |
| 225 | // the scheduler must not boot the good build there next. |
| 226 | Executed bool |
| 227 | } |
| 228 | |
| 229 | // The first-wake budget (spec §8.2). Vars so tests can shrink them. |
| 230 | var ( |
| 231 | firstWakeBudget = 55 * time.Second |
| 232 | firstWakeKillReserve = 25 * time.Second |
| 233 | firstWakeKillBound = 2 * time.Second |
| 234 | firstWakeSettle = 10 * time.Second |
| 235 | // firstWakeMinVerdict is the least health window B is exec'd with on a |
| 236 | // first wake that has a good build. B's window is fixed from the wake's |
| 237 | // start, so a slow lease, restore or seeding eats it; below this floor |
| 238 | // a failure would be the clock's, not the build's, and the good build |
| 239 | // boots instead with nothing recorded against B. |
| 240 | firstWakeMinVerdict = 20 * time.Second |
| 241 | // outcomeFenceWait bounds how long a new wake waits for a planned wake's |
| 242 | // outcome to be delivered before it plans (final review F3). OnGen's |
| 243 | // work is in memory plus one local failure-book write, so it normally |
| 244 | // returns in milliseconds; 5 s covers a slow disk with room to spare and |
| 245 | // still leaves the waiting requests most of their wakeWaitBound (30 s). |
| 246 | // It bounds the waiting request, never the restriction (final review |
| 247 | // wave 2, N1): past it the wake is refused before it plans, and the |
| 248 | // fence stays until the outcome is delivered. A later request retries. |
| 249 | outcomeFenceWait = 5 * time.Second |
| 250 | ) |
| 251 | |
| 252 | // genKiller (plan 3 Task 2, opset.go) is plan 2's generation kill |
| 253 | // (ExecBackend.KillGen and its SplitBackend forward): cgroup.kill on |
| 254 | // generation n's child cgroup, then wait until it is empty. This file uses |
| 255 | // it and declares none. |
| 256 | |
| 257 | // Current is the generation an awake instance serves, for the scheduler: a |
| 258 | // worker's is its captured spawn build (workerGenOf). A first wake under its |
| 259 | // 10 s observation reads as not awake, so the scheduler leaves it alone |
| 260 | // until the observation ends. |
| 261 | func (a *Service) Current(host string) (Gen, bool) { |
| 262 | a.mu.Lock() |
| 263 | defer a.mu.Unlock() |
| 264 | e := a.entries[host] |
| 265 | if e == nil || e.state != actAwake || e.firstWatch != nil { |
| 266 | return Gen{}, false |
| 267 | } |
| 268 | if e.worker { |
| 269 | return workerGenOf(e.workerServing), true |
| 270 | } |
| 271 | return e.gen, true |
| 272 | } |
| 273 | |
| 274 | // BounceFor is Bounce for a scheduled attempt: the event names the attempt's |
| 275 | // cause rather than "carlos restart" (spec §9.4). |
| 276 | func (a *Service) BounceFor(host string, cause Cause) bool { |
| 277 | a.mu.Lock() |
| 278 | defer a.mu.Unlock() |
| 279 | e := a.entries[host] |
| 280 | if e == nil || e.state != actAwake { |
| 281 | return false |
| 282 | } |
| 283 | emitEvent(a.Logs, e.route, "info", bounceText(e.worker, cause)) |
| 284 | a.beginStopLocked(e, stopWhyFor(cause)) |
| 285 | return true |
| 286 | } |
| 287 | |
| 288 | // stopWhyFor is the run-interruption reason (run spec §5.3) for a scheduled |
| 289 | // attempt's bounce: a new build for release, install and retry, a restart |
| 290 | // otherwise. |
| 291 | func stopWhyFor(cause Cause) string { |
| 292 | switch cause { |
| 293 | case CauseRelease, CauseInstall, CauseRetry: |
| 294 | return StopRelease |
| 295 | } |
| 296 | return StopRestart |
| 297 | } |
| 298 | |
| 299 | // bounceText is the one place BounceFor words its event. The words are |
| 300 | // internal/rolloutwords, approved copy. The flag-off literal in |
| 301 | // bounceLocked is never edited; restart's cause phrase keeps the two |
| 302 | // identical. |
| 303 | func bounceText(worker bool, cause Cause) string { |
| 304 | format := rolloutwords.EventBounceInstance |
| 305 | if worker { |
| 306 | format = rolloutwords.EventBounceSidecar |
| 307 | } |
| 308 | return fmt.Sprintf(format, rolloutwords.Cause(string(cause))) |
| 309 | } |
| 310 | |
| 311 | // killFailed ends a failed first-wake build outright (spec §8.2 step 1), |
| 312 | // bounded by firstWakeKillBound, and reports whether its end is proven. An |
| 313 | // instance generation is killed with cgroup.kill on its own child and waited |
| 314 | // empty (genKiller). Any error, errNoGenContainment included, is unproven: |
| 315 | // without a generation child there is no kill that reaches what B forked, |
| 316 | // and the good build must never start beside it. A worker is not a |
| 317 | // generation child (plan 2 D4): its proven stop, bounded the same, stands in |
| 318 | // for the kill, and THE termination proof (genTerminated on a Worker |
| 319 | // generation) empties and checks its host cgroup. Never called with a.mu |
| 320 | // held. |
| 321 | func (a *Service) killFailed(rt registry.Route, g Gen, p Proc, worker bool) error { |
| 322 | ctx, cancel := context.WithTimeout(context.Background(), firstWakeKillBound) |
| 323 | defer cancel() |
| 324 | if !worker { |
| 325 | k, ok := a.backend.(genKiller) |
| 326 | if !ok { |
| 327 | return fmt.Errorf("%s: this backend cannot kill a generation", rt.Host) |
| 328 | } |
| 329 | if err := k.KillGen(ctx, rt, g.N); err != nil { |
| 330 | return err |
| 331 | } |
| 332 | // The kill proved the end. The handle is still released through |
| 333 | // its Stop, the only close a flat child's control listener gets |
| 334 | // (backend_exec.go): left bound, it is the stale-socket trap |
| 335 | // (#129). Its own bound, since the kill may have spent this one. |
| 336 | sctx, scancel := context.WithTimeout(context.Background(), firstWakeKillBound) |
| 337 | defer scancel() |
| 338 | _ = p.Stop(sctx) |
| 339 | return nil |
| 340 | } |
| 341 | if err := p.Stop(ctx); err != nil { |
| 342 | return err |
| 343 | } |
| 344 | if !a.genTerminated(rt, Gen{N: g.N, Build: g.Build, Worker: true}, p) { |
| 345 | return ErrStillRunning |
| 346 | } |
| 347 | return nil |
| 348 | } |
| 349 | |
| 350 | // spawnPlanned is a planned wake's instance spawn (spec §8.2). The work |
| 351 | // before the exec (lease, restore, seed) has already run once and is shared. |
| 352 | // |
| 353 | // Every exec, B's and its fallback's, goes through startPinnedAt with |
| 354 | // planGate (PF7): the lease, the ownership and the plan's selection gate, |
| 355 | // asked at the exec. A refusal there runs nothing and is abandonment. |
| 356 | // |
| 357 | // Execution evidence comes from the spawn's result, as startPinnedChecked |
| 358 | // reads it: a returned process, or an unproven-start sentinel. Without it |
| 359 | // nothing of the build ran and nothing is recorded. With it, only what |
| 360 | // classifyStep calls the build's failure is recorded (a cancelled wake, a |
| 361 | // stale attempt and a mismatched exec are not), and only for a first wake. |
| 362 | // |
| 363 | // For a first wake in overlap mode with a good build, B's health window |
| 364 | // ends at the wake's start plus 55 s minus 25 s, a failed B is killed |
| 365 | // outright, and the good build gets whatever time remains. When B's end is |
| 366 | // not proven, nothing more starts. When less than firstWakeMinVerdict of |
| 367 | // B's window is left at its exec (a slow lease, restore or seeding), B is |
| 368 | // not exec'd at all: the good build boots instead and nothing is recorded, |
| 369 | // since a health check cut that short would fail B for the clock's sake. |
| 370 | func (a *Service) spawnPlanned(ctx context.Context, e *actEntry, rt registry.Route, plan WakePlan, w *wokenInstance, lease Lease) (Proc, string, Gen, error) { |
| 371 | plan, err := a.checkPlan(ctx, e, rt, plan) |
| 372 | if err != nil { |
| 373 | return nil, "", Gen{}, err |
| 374 | } |
| 375 | canFall := plan.FirstWake && plan.Fallback != nil && plan.Mode == ModeOverlap |
| 376 | boot := plan.Boot |
| 377 | gate := a.planGate(ctx, e, lease, plan, boot.Version) |
| 378 | var healthBy time.Time |
| 379 | var left time.Duration |
| 380 | if canFall { |
| 381 | healthBy = e.wakeStart.Add(firstWakeBudget - firstWakeKillReserve) |
| 382 | gate = verdictFloorGate(gate, healthBy, &left) |
| 383 | } |
| 384 | // Every exec here that runs the good build in another build's place is |
| 385 | // recorded with the planner's selection (ruling R-PLANNED-RECOVERY): a |
| 386 | // recovery boot (normally recorded at step 2b already), the good build |
| 387 | // booted untried and the fallback. Each exec's own context only. |
| 388 | proc, ver, g, err := a.startPinnedAt(withRecoverySelection(ctx, plan, rt, boot.Version), rt, genOf(boot), gate, healthBy) |
| 389 | w.bootRan = err == nil || startMayHaveRun(proc, err) |
| 390 | if proc == nil && errors.Is(err, errNoTimeForVerdict) { |
| 391 | // B never ran: too little of its window was left for a verdict at |
| 392 | // the exec. The good build boots in its place through the same |
| 393 | // gate, and the wake is reported as the good build's ordinary wake |
| 394 | // (w.untried): no failure of B, no fallback, no observation. |
| 395 | // Operator log only; nothing happened to the member's build. |
| 396 | good := *plan.Fallback |
| 397 | log.Printf("activator: %s: first wake of %s had %v of its health window left at the exec, under the %v floor; booting the good build %s without trying it", |
| 398 | rt.Host, boot.Version, left.Round(time.Millisecond), firstWakeMinVerdict, good.Version) |
| 399 | w.untried = true |
| 400 | gctx := WithPlannedSelection(ctx, PlannedSelection{Build: good.Version, InPlaceOf: boot.Version}) |
| 401 | gproc, gver, gg, gerr := a.startPinnedAt(gctx, rt, genOf(good), a.planGate(ctx, e, lease, plan, good.Version), time.Time{}) |
| 402 | if gerr != nil { |
| 403 | return gproc, gver, gg, fmt.Errorf("the good build %s, booted without trying %s: %w", good.Version, boot.Version, gerr) |
| 404 | } |
| 405 | return gproc, gver, withIdentity(gg, good), nil |
| 406 | } |
| 407 | if err == nil { |
| 408 | return proc, ver, withIdentity(g, boot), nil |
| 409 | } |
| 410 | // A nested start root accepted whose outcome is uncertain, with no |
| 411 | // process back, may have run B (Astra R3 ruling 206, zdd-readiness |
| 412 | // Task 6). In hold mode that is always B's failure: nothing falls back |
| 413 | // from a hold, and a wake reported as cancelled would leave the build |
| 414 | // free to boot again. In overlap mode only a unit that exited before it |
| 415 | // was observed (errStartedThenExited) is a proven crash at the start and |
| 416 | // fails B; any other uncertain overlap start stays "may have run" |
| 417 | // (afterWake's GenCancelled with Executed), unless its kill is |
| 418 | // unproven, which fails B below as before. |
| 419 | uncertain := proc == nil && startMayHaveRun(nil, err) |
| 420 | crashed := uncertain && errors.Is(err, errStartedThenExited) |
| 421 | uncertainFailure := uncertain && (plan.Mode == ModeHold || crashed) |
| 422 | executed := proc != nil || genStartUnproven(err) |
| 423 | if !uncertainFailure && (!executed || classifyStep(ctx, err) != stepFailed || errors.Is(err, errPinMismatch)) { |
| 424 | // Nothing of the build ran, or the wake's lifecycle ended it |
| 425 | // (maintenance, Shutdown, a stop), or its gate refused, or the |
| 426 | // backend ran something else: no failure of the build (spec §6.4, |
| 427 | // §7.4). Nothing is recorded and nothing falls back; doWake's |
| 428 | // cleanup stops a returned process and proves it gone. |
| 429 | return proc, ver, g, err |
| 430 | } |
| 431 | if plan.FirstWake { |
| 432 | w.failed, w.failReason = &boot, err.Error() |
| 433 | } |
| 434 | if !canFall { |
| 435 | return proc, ver, g, err |
| 436 | } |
| 437 | if proc == nil { |
| 438 | // No handle to kill. The good build starts only beside a B whose |
| 439 | // end is proven (plan review finding 9): a crash at the start that |
| 440 | // the backend's own kill proved gone (no errGenUnproven in the |
| 441 | // chain). Every other overlap start that reaches here with no |
| 442 | // process carries errGenUnproven: its end is unproven, so nothing |
| 443 | // more starts, the wake serves nothing, and doWake keeps the lease |
| 444 | // and the generation as an unproven wake. |
| 445 | if !crashed || genStartUnproven(err) { |
| 446 | return proc, ver, g, err |
| 447 | } |
| 448 | } else if kerr := a.killFailed(rt, g, proc, false); kerr != nil { |
| 449 | return proc, ver, g, fmt.Errorf("%w; its end was not proven, so the good build was not started: %v", err, kerr) |
| 450 | } |
| 451 | good := *plan.Fallback |
| 452 | fctx := WithPlannedSelection(ctx, PlannedSelection{Build: good.Version, InPlaceOf: boot.Version}) |
| 453 | fproc, fver, fg, ferr := a.startPinnedAt(fctx, rt, genOf(good), a.planGate(ctx, e, lease, plan, good.Version), time.Time{}) |
| 454 | if ferr != nil { |
| 455 | if fproc == nil && !genStartUnproven(ferr) { |
| 456 | // Nothing of the good build ran (its gate refused: a fence, a |
| 457 | // shutdown, a lost lease or entry since B's exec). B's failure is |
| 458 | // still recorded (w.failed); its process, proven gone, is what |
| 459 | // doWake's cleanup proves once more. |
| 460 | return proc, ver, g, fmt.Errorf("%w; the good build was not started: %v", err, ferr) |
| 461 | } |
| 462 | return fproc, fver, fg, fmt.Errorf("%w; the good build %s failed too: %w", err, good.Version, ferr) |
| 463 | } |
| 464 | w.fellBack = true |
| 465 | return fproc, fver, withIdentity(fg, good), nil |
| 466 | } |
| 467 | |
| 468 | // errNoTimeForVerdict: at B's exec, less than firstWakeMinVerdict of its |
| 469 | // health window was left. B is not exec'd; it is never a failure of B. |
| 470 | var errNoTimeForVerdict = errors.New("too little of the first wake's health window is left for a verdict") |
| 471 | |
| 472 | // verdictFloorGate is gate with the first-wake floor asked after it, at |
| 473 | // the exec: B's window is the earlier of exec + spawnHealthTimeout and |
| 474 | // healthBy, less now. Below firstWakeMinVerdict it refuses with |
| 475 | // errNoTimeForVerdict and stores the window in *left for the log. The |
| 476 | // ownership, lease and plan checks go first, so their refusal (abandonment) |
| 477 | // is never mistaken for a lack of time. |
| 478 | func verdictFloorGate(gate func() error, healthBy time.Time, left *time.Duration) func() error { |
| 479 | return func() error { |
| 480 | if err := gate(); err != nil { |
| 481 | return err |
| 482 | } |
| 483 | now := time.Now() |
| 484 | end := now.Add(spawnHealthTimeout) |
| 485 | if healthBy.Before(end) { |
| 486 | end = healthBy |
| 487 | } |
| 488 | if *left = end.Sub(now); *left < firstWakeMinVerdict { |
| 489 | return errNoTimeForVerdict |
| 490 | } |
| 491 | return nil |
| 492 | } |
| 493 | } |
| 494 | |
| 495 | // spawnPlannedWorker is spawnPlanned for a socketless child (spec §6.5). |
| 496 | // Each spawn uses startWorker's contract exactly as doWake's worker branch |
| 497 | // does (PF8): the build is published before its exec, planGate is asked at |
| 498 | // the exec (after the publication), and on success the captured identity is |
| 499 | // installed and the pending publication dropped in one critical section, |
| 500 | // then published again. A backend without the staged capability cannot |
| 501 | // exec a named build, so a planned worker is refused there before anything |
| 502 | // runs (errSwapNoGens). plannedWorkerReady stands in for the health check, |
| 503 | // and a failed release-cause first wake in overlap mode is replaced by the |
| 504 | // good build once the failed child's host cgroup is proven empty. |
| 505 | func (a *Service) spawnPlannedWorker(ctx context.Context, e *actEntry, rt registry.Route, plan WakePlan, env map[string]string, w *wokenInstance, lease Lease) (Proc, release.StagedIdentity, error) { |
| 506 | if _, ok := a.backend.(stagedWorkerStarter); !ok { |
| 507 | return nil, release.StagedIdentity{}, errSwapNoGens |
| 508 | } |
| 509 | plan, err := a.checkPlan(ctx, e, rt, plan) |
| 510 | if err != nil { |
| 511 | return nil, release.StagedIdentity{}, err |
| 512 | } |
| 513 | start := func(sctx context.Context, id release.StagedIdentity) (Proc, release.StagedIdentity, error) { |
| 514 | p, got, pend, err := a.startWorker(sctx, rt, env, &id, a.planGate(ctx, e, lease, plan, id.Version)) |
| 515 | if err != nil { |
| 516 | return p, got, err |
| 517 | } |
| 518 | a.mu.Lock() |
| 519 | e.workerServing = got |
| 520 | a.dropPendingGenerationLocked(pend) |
| 521 | a.mu.Unlock() |
| 522 | _ = a.persistGenerations() |
| 523 | if got.StagedPath != id.StagedPath { |
| 524 | return p, got, fmt.Errorf("%w: worker build %s from %q; the plan chose %s from %q", errPinMismatch, got.Version, got.StagedPath, id.Version, id.StagedPath) |
| 525 | } |
| 526 | return p, got, plannedWorkerReady(ctx, p) |
| 527 | } |
| 528 | canFall := plan.FirstWake && plan.Fallback != nil && plan.Mode == ModeOverlap |
| 529 | boot := plan.Boot |
| 530 | proc, id, err := start(withRecoverySelection(ctx, plan, rt, boot.Version), boot) |
| 531 | w.bootRan = err == nil || startMayHaveRun(proc, err) |
| 532 | if err == nil || proc == nil { |
| 533 | // Ready, or nothing of the build ran: a refusal at the gate, an |
| 534 | // unpublished build (ErrBuildNotPublished), a spawn error with no |
| 535 | // child. |
| 536 | return proc, id, err |
| 537 | } |
| 538 | if classifyStep(ctx, err) != stepFailed || errors.Is(err, errPinMismatch) { |
| 539 | // A cancelled readiness window is the wake's lifecycle, not the |
| 540 | // build (spec §6.4, §6.5): nothing recorded, nothing falls back. |
| 541 | return proc, id, err |
| 542 | } |
| 543 | if plan.FirstWake { |
| 544 | w.failed, w.failReason = &boot, err.Error() |
| 545 | } |
| 546 | if !canFall { |
| 547 | return proc, id, err |
| 548 | } |
| 549 | if kerr := a.killFailed(rt, Gen{Build: id.Version}, proc, true); kerr != nil { |
| 550 | return proc, id, fmt.Errorf("%w; its end was not proven, so the good build was not started: %v", err, kerr) |
| 551 | } |
| 552 | // The fallback is recorded with the planner's selection (ruling |
| 553 | // R-PLANNED-RECOVERY), as spawnPlanned's is. |
| 554 | fproc, fid, ferr := start(WithPlannedSelection(ctx, PlannedSelection{Build: plan.Fallback.Version, InPlaceOf: boot.Version}), *plan.Fallback) |
| 555 | if ferr != nil { |
| 556 | if fproc == nil { |
| 557 | // Nothing of the good build ran: B's failure is still recorded, |
| 558 | // and B's child, proven gone, is what doWake's cleanup proves. |
| 559 | return proc, id, fmt.Errorf("%w; the good build was not started: %v", err, ferr) |
| 560 | } |
| 561 | return fproc, fid, fmt.Errorf("%w; the good build %s failed too: %w", err, plan.Fallback.Version, ferr) |
| 562 | } |
| 563 | w.fellBack = true |
| 564 | return fproc, fid, nil |
| 565 | } |
| 566 | |
| 567 | // plannedWorkerReady is spec §6.5's readiness for a socketless planned |
| 568 | // wake, the rule ReplaceWorker applies too: the child is ready once it is |
| 569 | // still running firstWakeSettle (10 s) after it started, and a clean exit |
| 570 | // inside that window is a one-shot's success. Any other exit fails. An early |
| 571 | // ready report does not shorten the window. The flag-off wake keeps |
| 572 | // awaitWorkerSettle (ready report, sidecarSettle, every early exit a |
| 573 | // failure), unchanged. |
| 574 | func plannedWorkerReady(ctx context.Context, p Proc) error { |
| 575 | switch observeSettle(ctx, p, firstWakeSettle) { |
| 576 | case settleOK: |
| 577 | return nil |
| 578 | case settleExited: |
| 579 | if !procCrashed(p) { |
| 580 | return nil |
| 581 | } |
| 582 | return errWorkerExitedEarly |
| 583 | } |
| 584 | if err := ctx.Err(); err != nil { |
| 585 | return err |
| 586 | } |
| 587 | return context.Canceled |
| 588 | } |
| 589 | |
| 590 | // outcomeFence counts a host's planned-wake outcomes that have been taken |
| 591 | // but not yet delivered to OnGen (final review F3). done closes when the |
| 592 | // count returns to zero. |
| 593 | type outcomeFence struct { |
| 594 | n int |
| 595 | done chan struct{} |
| 596 | } |
| 597 | |
| 598 | // holdOutcomeLocked takes one of host's outcome fences and returns its |
| 599 | // release. The release takes a.mu itself, so it is called off a.mu, after |
| 600 | // the outcome's OnGen has returned; only its first call counts. The caller |
| 601 | // holds a.mu. |
| 602 | func (a *Service) holdOutcomeLocked(host string) func() { |
| 603 | if a.outcomeFences == nil { |
| 604 | a.outcomeFences = make(map[string]*outcomeFence) |
| 605 | } |
| 606 | f := a.outcomeFences[host] |
| 607 | if f == nil { |
| 608 | f = &outcomeFence{done: make(chan struct{})} |
| 609 | a.outcomeFences[host] = f |
| 610 | } |
| 611 | f.n++ |
| 612 | var once sync.Once |
| 613 | return func() { |
| 614 | once.Do(func() { |
| 615 | a.mu.Lock() |
| 616 | defer a.mu.Unlock() |
| 617 | if f.n--; f.n == 0 { |
| 618 | close(f.done) |
| 619 | if a.outcomeFences[host] == f { |
| 620 | delete(a.outcomeFences, host) |
| 621 | } |
| 622 | } |
| 623 | }) |
| 624 | } |
| 625 | } |
| 626 | |
| 627 | // errOutcomeUndelivered refuses a wake, before it plans, while an earlier |
| 628 | // planned wake's outcome is still being delivered past outcomeFenceWait |
| 629 | // (final review wave 2, N1). The fence fails closed: planning without the |
| 630 | // outcome could boot a build whose failure (a hold's included) is not yet |
| 631 | // recorded. Members get the maintenance fence's existing text |
| 632 | // (memberSafeWakeReason): the platform holds the host closed until its own |
| 633 | // work is done, and it reopens by itself. |
| 634 | var errOutcomeUndelivered = errors.New("an earlier planned wake's outcome is still being delivered") |
| 635 | |
| 636 | // awaitOutcomes is a new wake's wait, before it plans, for host's |
| 637 | // planned-wake outcomes still being delivered (final review F3; spec §8.1, |
| 638 | // O7): a failed first wake's entry goes before afterWake's OnGen records |
| 639 | // the failure, and a first wake's observer reaches its verdict before |
| 640 | // OnGen hears it. Planning in that gap would pick the failed build again, |
| 641 | // with no record yet to hold it. The wait is bounded by outcomeFenceWait |
| 642 | // and the wake's own context; the fence is not (final review wave 2, N1). |
| 643 | // A wait that ends before the delivery refuses the wake: an error means |
| 644 | // the caller must not plan or launch anything. The fence stays until the |
| 645 | // outcome's OnGen returns, so the next request waits again. Called off |
| 646 | // a.mu. |
| 647 | func (a *Service) awaitOutcomes(ctx context.Context, host string) error { |
| 648 | a.mu.Lock() |
| 649 | f := a.outcomeFences[host] |
| 650 | a.mu.Unlock() |
| 651 | if f == nil { |
| 652 | return nil |
| 653 | } |
| 654 | bound := time.NewTimer(outcomeFenceWait) |
| 655 | defer bound.Stop() |
| 656 | select { |
| 657 | case <-f.done: |
| 658 | return nil |
| 659 | case <-ctx.Done(): |
| 660 | // Refused the same way: the outcome is still undelivered. |
| 661 | return fmt.Errorf("%w: %w", errOutcomeUndelivered, ctx.Err()) |
| 662 | case <-bound.C: |
| 663 | log.Printf("activator: %s: an earlier planned wake's outcome is still being delivered after %v; refusing this wake until it is", host, outcomeFenceWait) |
| 664 | return errOutcomeUndelivered |
| 665 | } |
| 666 | } |
| 667 | |
| 668 | // outcomeOwedLocked reports whether wake w of e ends with a failure that |
| 669 | // afterWake will deliver (GenFailed, GenFellBack): the outcome fence covers |
| 670 | // it. The caller holds a.mu. |
| 671 | func (a *Service) outcomeOwedLocked(e *actEntry, w *wokenInstance, err error) bool { |
| 672 | return a.OnGen != nil && w != nil && e.plan.planned() && (w.failed != nil || (err != nil && w.holdRanUnjudged(e.plan))) |
| 673 | } |
| 674 | |
| 675 | // holdRanUnjudged reports whether a failed wake of plan ran (or may have |
| 676 | // run) its first wake's hold build without a verdict on it (final review |
| 677 | // wave 2, N2): its lifecycle ended it after the exec (a stop, a fence, a |
| 678 | // step after the spawn), or its start's outcome is uncertain. afterWake |
| 679 | // reports it as GenCancelled with Executed, and the outcome fence covers |
| 680 | // the delivery, so the next wake plans only once the scheduler holds the |
| 681 | // instance off its good build. |
| 682 | func (w *wokenInstance) holdRanUnjudged(plan WakePlan) bool { |
| 683 | return w.bootRan && w.failed == nil && plan.FirstWake && plan.Mode == ModeHold |
| 684 | } |
| 685 | |
| 686 | // emitPlanned is emitGen for an event of a wake that ran plan: it carries |
| 687 | // the plan's token and config revision (final review F4), so every event of |
| 688 | // one planned wake, on every path, names the same wake. Never called with |
| 689 | // a.mu held (emitGen's rule). |
| 690 | func (a *Service) emitPlanned(plan WakePlan, ev GenEvent) { |
| 691 | ev.Token, ev.ConfigRev = plan.Token, plan.ConfigRev |
| 692 | a.emitGen(ev) |
| 693 | } |
| 694 | |
| 695 | // emitGen hands ev to OnGen. It is never called with a.mu held: OnGen |
| 696 | // reaches the scheduler's served, publishServing and writeGood, which take |
| 697 | // its goodMu and call back into Current. |
| 698 | func (a *Service) emitGen(ev GenEvent) { |
| 699 | if a.OnGen != nil { |
| 700 | a.OnGen(ev) |
| 701 | } |
| 702 | } |
| 703 | |
| 704 | // cancelPlanned reports that a planned wake of plan ended with no verdict |
| 705 | // on its build and nothing owed to the outcome fence (rollout Task 5, plan |
| 706 | // review #1): refused at its lease, its exec or its check, replaced by a |
| 707 | // replan, or (ran true) an overlap or ordinary wake its lifecycle ended |
| 708 | // after the exec. It records nothing; it is the wake's terminal event, which |
| 709 | // ends the scheduler's write-ahead intent for it. Only a plan with a token |
| 710 | // is reported, and only to an OnGen that is set, so a box with the flag off |
| 711 | // hears nothing. Never called with a.mu held (emitGen's rule). |
| 712 | func (a *Service) cancelPlanned(rt registry.Route, plan WakePlan, ran bool) { |
| 713 | if a.OnGen == nil || plan.Token == 0 || !plan.planned() { |
| 714 | return |
| 715 | } |
| 716 | ev := GenEvent{Kind: GenCancelled, Route: rt, Mode: plan.Mode, FirstWake: plan.FirstWake, At: time.Now(), Executed: ran} |
| 717 | if ran { |
| 718 | ev.Gen = genOf(plan.Boot) |
| 719 | } |
| 720 | a.emitPlanned(plan, ev) |
| 721 | } |
| 722 | |
| 723 | // afterWake reports a planned wake's outcome. wake() calls it after |
| 724 | // e.ready has closed, without a.mu (emitGen's rule). |
| 725 | // |
| 726 | // Every planned wake with a token ends with exactly one terminal event for |
| 727 | // it (rollout Task 5): its failure, its fallback, its verdict or its |
| 728 | // cancellation here, or the first-wake observer's. A failed wake that owed |
| 729 | // nothing to the outcome fence (outcomeOwedLocked) reports its cancellation |
| 730 | // (cancelPlanned) after its entry is gone and admission has reopened; that |
| 731 | // is safe because the event records nothing, and a newer wake's intent |
| 732 | // replaces this one's, so its late arrival ends nothing it should not. |
| 733 | func (a *Service) afterWake(e *actEntry, w *wokenInstance, err error) { |
| 734 | plan := e.plan // written only by this wake's own goroutine |
| 735 | if a.OnGen == nil || !plan.planned() { |
| 736 | return |
| 737 | } |
| 738 | if w == nil { |
| 739 | // No result at all (a worker's token not minted, a claimed wake that |
| 740 | // ran nothing of the planned build): nothing of it ran. |
| 741 | if err != nil { |
| 742 | a.cancelPlanned(e.route, plan, false) |
| 743 | } |
| 744 | return |
| 745 | } |
| 746 | now := time.Now() |
| 747 | switch { |
| 748 | case err != nil: |
| 749 | switch { |
| 750 | case w.failed != nil: |
| 751 | a.emitPlanned(plan, GenEvent{Kind: GenFailed, Route: e.route, Failed: *w.failed, Mode: plan.Mode, FirstWake: true, Reason: w.failReason, At: now, FailedUnder: plan.FailedUnder}) |
| 752 | case w.holdRanUnjudged(plan): |
| 753 | a.emitPlanned(plan, GenEvent{Kind: GenCancelled, Route: e.route, Gen: genOf(plan.Boot), Mode: plan.Mode, FirstWake: true, At: now, Executed: true}) |
| 754 | default: |
| 755 | // Refused before the exec (its lease, a fence, its check), or an |
| 756 | // overlap or ordinary wake ended after the exec with no verdict. |
| 757 | a.cancelPlanned(e.route, plan, w.bootRan) |
| 758 | } |
| 759 | case w.fellBack: |
| 760 | a.emitPlanned(plan, GenEvent{Kind: GenFellBack, Route: e.route, Gen: w.planned, Failed: *w.failed, Mode: plan.Mode, FirstWake: true, Reason: w.failReason, At: now, FailedUnder: plan.FailedUnder}) |
| 761 | case w.untried: |
| 762 | // B was never exec'd (too little time for a verdict) and the good |
| 763 | // build serves: reported exactly as a healthy wake of the good build |
| 764 | // that is not a first wake, so the scheduler records it as served |
| 765 | // and B stays a candidate for the next wake. |
| 766 | a.emitPlanned(plan, GenEvent{Kind: GenHealthy, Route: e.route, Gen: w.planned, Mode: plan.Mode, At: now}) |
| 767 | default: |
| 768 | a.emitPlanned(plan, GenEvent{Kind: GenHealthy, Route: e.route, Gen: w.planned, Mode: plan.Mode, FirstWake: plan.FirstWake, At: now}) |
| 769 | switch { |
| 770 | case plan.FirstWake && e.worker: |
| 771 | // A socketless first wake's readiness window (plannedWorkerReady, |
| 772 | // spec §6.5) was its settle: it is settled now. A second window |
| 773 | // would fail a build that was already ready, where §8.6 calls for |
| 774 | // ordinary crash recovery. |
| 775 | a.emitPlanned(plan, GenEvent{Kind: GenSettled, Route: e.route, Gen: w.planned, Mode: plan.Mode, FirstWake: true, At: now}) |
| 776 | case plan.FirstWake: |
| 777 | a.watchFirstWake(e, w.planned, plan) |
| 778 | } |
| 779 | } |
| 780 | } |
| 781 | |
| 782 | // armFirstWatchLocked registers a first wake's observation in the awake |
| 783 | // transition, before e.ready closes (spec §6.4): Current hides the |
| 784 | // generation from the instant it is reachable, and an exit or a stop |
| 785 | // before the observer goroutine runs is classified by it, never lost. |
| 786 | // wake() calls it after e.gen is set, and before the fence and bounce |
| 787 | // checks there, so a stop they begin cancels the observation. It also takes |
| 788 | // the host's outcome fence (final review F3), which watchFirstWake releases |
| 789 | // once its terminal event has been delivered: the entry can be stopped and |
| 790 | // a new wake begun before then, and that wake must not plan until the |
| 791 | // verdict is recorded. The caller holds a.mu. |
| 792 | func (a *Service) armFirstWatchLocked(e *actEntry, w *wokenInstance) { |
| 793 | if a.Planner == nil || a.OnGen == nil || e.worker || !e.plan.planned() || !e.plan.FirstWake || w.fellBack || w.failed != nil || w.untried { |
| 794 | return |
| 795 | } |
| 796 | ctx, cancel := context.WithCancel(context.Background()) |
| 797 | e.firstWatch, e.firstWatchCtx = cancel, ctx |
| 798 | e.firstWatchRelease = a.holdOutcomeLocked(e.host) |
| 799 | } |
| 800 | |
| 801 | // watchFirstWake is a first wake's 10 s observation (spec §6.4), on plan 3's |
| 802 | // observeSettle. The waiting requests do not sit through it. An intentional |
| 803 | // stop cancels it (beginStopLocked) and records nothing; an unexpected exit |
| 804 | // fails the build. |
| 805 | func (a *Service) watchFirstWake(e *actEntry, g Gen, plan WakePlan) { |
| 806 | cancelled := func(rt registry.Route) { |
| 807 | // Every observation GenHealthy opened is closed by an event; this |
| 808 | // one says "no verdict", so the scheduler stops showing the host |
| 809 | // as settling and records nothing (spec §6.4, §9.1). The build was |
| 810 | // healthy, so it ran (Executed). |
| 811 | a.emitPlanned(plan, GenEvent{Kind: GenCancelled, Route: rt, Gen: g, Mode: plan.Mode, FirstWake: true, At: time.Now(), Executed: true}) |
| 812 | } |
| 813 | // The observation was registered in the awake transition |
| 814 | // (armFirstWatchLocked), so a stop that began meanwhile has already |
| 815 | // cancelled it, or, for a crash, marked unexpectedExit: observeSettle and |
| 816 | // the classification below decide either way. Nothing registered means |
| 817 | // no first-wake observation for this wake. |
| 818 | a.mu.Lock() |
| 819 | ctx, rt, proc := e.firstWatchCtx, e.route, e.proc |
| 820 | a.mu.Unlock() |
| 821 | if ctx == nil { |
| 822 | cancelled(rt) |
| 823 | return |
| 824 | } |
| 825 | go func() { |
| 826 | r := observeSettle(ctx, proc, firstWakeSettle) |
| 827 | a.mu.Lock() |
| 828 | rt, unexpected, release := e.route, e.unexpectedExit, e.firstWatchRelease |
| 829 | a.mu.Unlock() |
| 830 | // end closes the observation: Current shows the generation again, |
| 831 | // and a stop no longer has an observation to cancel. |
| 832 | end := func() { |
| 833 | a.mu.Lock() |
| 834 | cancel := e.firstWatch |
| 835 | e.firstWatch, e.firstWatchCtx, e.firstWatchRelease = nil, nil, nil |
| 836 | a.mu.Unlock() |
| 837 | if cancel != nil { |
| 838 | cancel() |
| 839 | } |
| 840 | } |
| 841 | // The verdict is delivered before the host's outcome fence is |
| 842 | // released (final review F3): a new wake that began meanwhile plans |
| 843 | // only once the scheduler has recorded it. |
| 844 | defer func() { |
| 845 | if release != nil { |
| 846 | release() |
| 847 | } |
| 848 | }() |
| 849 | exited := r == settleExited || (r == settleCancelled && unexpected) |
| 850 | switch { |
| 851 | case r == settleOK: |
| 852 | // Ended first: the scheduler records a settled build as good |
| 853 | // only while Current shows it (rollout's acknowledged). |
| 854 | end() |
| 855 | a.emitPlanned(plan, GenEvent{Kind: GenSettled, Route: rt, Gen: g, Mode: plan.Mode, FirstWake: true, At: time.Now()}) |
| 856 | case exited: |
| 857 | // Delivered first, so nothing sees the failed generation as |
| 858 | // current (and admits an attempt onto it) before the failure is |
| 859 | // recorded. |
| 860 | a.emitPlanned(plan, GenEvent{Kind: GenFailed, Route: rt, Failed: plan.Boot, Mode: plan.Mode, FirstWake: true, |
| 861 | Reason: "exited within " + firstWakeSettle.String() + " of becoming healthy", At: time.Now(), FailedUnder: plan.FailedUnder}) |
| 862 | end() |
| 863 | default: // an intentional stop cancelled it |
| 864 | cancelled(rt) |
| 865 | end() |
| 866 | } |
| 867 | }() |
| 868 | } |
| 869 | |
| 870 | // claimedPlanResult carries a claims-on planned wake's outcome to afterWake. |
| 871 | // Plan 3's startAdmitted records the pinned build it saw fail |
| 872 | // (e.pinFailed) and, once the isolated launch opens, starts the good build |
| 873 | // in its place (StartPolicy.Fallback). w is the claim's result, nil when |
| 874 | // the wake failed. The returned value is what doClaimWake hands back to |
| 875 | // wake(). On this base every claims-on launch refuses before its exec hook, |
| 876 | // so pinFailed stays nil and a planned claimed wake reports nothing. |
| 877 | func (a *Service) claimedPlanResult(e *actEntry, w *wokenInstance) *wokenInstance { |
| 878 | a.mu.Lock() |
| 879 | failed, why, plan := e.pinFailed, e.pinFailReason, e.plan |
| 880 | a.mu.Unlock() |
| 881 | if !plan.planned() { |
| 882 | return w |
| 883 | } |
| 884 | if w == nil { |
| 885 | if failed == nil || !plan.FirstWake { |
| 886 | return nil // nothing of the planned build ran: no failure of the build |
| 887 | } |
| 888 | return &wokenInstance{failed: &plan.Boot, failReason: why} |
| 889 | } |
| 890 | served := plan.Boot |
| 891 | if failed != nil && plan.FirstWake && plan.Fallback != nil && w.spawnVersion == plan.Fallback.Version { |
| 892 | served = *plan.Fallback |
| 893 | w.failed, w.failReason, w.fellBack = &plan.Boot, why, true |
| 894 | } |
| 895 | w.planned = genOf(served) |
| 896 | w.planned.N, w.planned.Socket, w.planned.Control = w.gen.N, w.gen.Socket, w.gen.Control |
| 897 | return w |
| 898 | } |
| 899 | |
amadan