| 1 | // swap.go is the overlap and hold swap (zero-downtime deploys spec §7.4, |
| 2 | // §7.5, §7.8): one attempt moves one awake instance to a new generation. |
| 3 | // Nothing calls Service.Swap until plan 5 wires CARLOS_SWAP. With plan 1's |
| 4 | // generations switch off, it refuses every attempt and spawns nothing. |
| 5 | package activator |
| 6 | |
| 7 | import ( |
| 8 | "context" |
| 9 | "errors" |
| 10 | "fmt" |
| 11 | "log" |
| 12 | "time" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/cgroup" |
| 15 | "github.com/carlosframework/platform/internal/hostbox" |
| 16 | "github.com/carlosframework/platform/internal/hostguard" |
| 17 | "github.com/carlosframework/platform/internal/hostwrite" |
| 18 | "github.com/carlosframework/platform/internal/instancectl" |
| 19 | "github.com/carlosframework/platform/internal/registry" |
| 20 | ) |
| 21 | |
| 22 | // Cause is why an attempt exists (spec §6.3). |
| 23 | type Cause string |
| 24 | |
| 25 | const ( |
| 26 | CauseRelease Cause = "release" |
| 27 | CauseInstall Cause = "install" |
| 28 | CauseRetry Cause = "retry" |
| 29 | CauseRestart Cause = "restart" |
| 30 | CauseConfig Cause = "config" |
| 31 | ) |
| 32 | |
| 33 | // SwapMode is how an awake instance moves to a new generation (spec §4.1). |
| 34 | type SwapMode string |
| 35 | |
| 36 | const ( |
| 37 | ModeOverlap SwapMode = "overlap" |
| 38 | ModeHold SwapMode = "hold" |
| 39 | ) |
| 40 | |
| 41 | // Attempt is one try at moving one instance to a new generation (spec §6.3). |
| 42 | type Attempt struct { |
| 43 | Host string |
| 44 | Cause Cause |
| 45 | Build string |
| 46 | StagedPath string |
| 47 | Digest string // expected sha256 of StagedPath, verified before exec |
| 48 | Demand string // release demand for the artifact: "hold" or "" |
| 49 | Mode SwapMode |
| 50 | ConfigRev string |
| 51 | RestartID string |
| 52 | SchedRev uint64 // scheduler revision; the newest-attempt check compares it |
| 53 | Canary bool // observe the 10 s settle |
| 54 | } |
| 55 | |
| 56 | // SwapOutcome is how an attempt ended. |
| 57 | type SwapOutcome string |
| 58 | |
| 59 | const ( |
| 60 | SwapDone SwapOutcome = "done" // switched (overlap) or replaced (hold) |
| 61 | SwapFailed SwapOutcome = "failed" // successor failed health or settle; incumbent still serving (overlap) |
| 62 | SwapAbandoned SwapOutcome = "abandoned" // superseded or a re-check failed; not a build failure |
| 63 | SwapDown SwapOutcome = "down" // hold mode failed: nothing serving |
| 64 | ) |
| 65 | |
| 66 | // SwapResult is Swap's answer. Reason is diagnostic text for the |
| 67 | // scheduler (plan 5); member wording is plan 6's. |
| 68 | type SwapResult struct { |
| 69 | Outcome SwapOutcome |
| 70 | Gen Gen // the generation now current (on SwapDone) |
| 71 | Reason string |
| 72 | // Drain is how long the retiring generation took from SIGTERM to its |
| 73 | // proven exit (an overlap), or the old generation's stop (a hold). Zero |
| 74 | // when nothing retired. The "swapped" event reports it (spec §9.4). |
| 75 | Drain time.Duration |
| 76 | // Mode is the mode the attempt actually ran in. It differs from |
| 77 | // Attempt.Mode when an overlap attempt on a foreign-layout row ran as a |
| 78 | // hold (spec §2.9): the scheduler records a failure under this mode, so |
| 79 | // a failed hold is held down (spec §6.4, §8.3). Swap sets it. |
| 80 | Mode SwapMode |
| 81 | // Executed is the attempt's execution evidence (final review F8), one |
| 82 | // side of a tri-state (final review wave 2, N4): false means it is |
| 83 | // PROVEN that nothing of the attempt's build ran; true means it ran OR |
| 84 | // it is not known whether it ran. True once a process of the build |
| 85 | // started, whatever happened to it after (an overlap successor, a |
| 86 | // hold's pinned wake, a worker's respawn child, installed or shed), and |
| 87 | // also when a start was handed off and failed with its outcome |
| 88 | // uncertain (startMayHaveRun) or when ReplaceWorker could not learn |
| 89 | // what its respawn did. Only false lets an abandoned attempt count as |
| 90 | // having spent nothing of what it was granted: the scheduler re-arms a |
| 91 | // `carlos instances retry` allowance on it. Swap and ReplaceWorker set |
| 92 | // it; every other result leaves it false, and each of those returns |
| 93 | // before anything of the build could start. |
| 94 | Executed bool |
| 95 | // CleanExit is set by ReplaceWorker on a SwapDone whose replacement's |
| 96 | // build ran to a clean exit inside the settle window: a one-shot's |
| 97 | // success (spec §6.5). It has no generation the activator will go on |
| 98 | // serving, so nothing acknowledges it afterwards; the scheduler records |
| 99 | // it as the good build from this result (spec §8.5, Astra R3 flag |
| 100 | // triage T9). A replacement still running at the end of the window is |
| 101 | // SwapDone with CleanExit false. |
| 102 | CleanExit bool |
| 103 | } |
| 104 | |
| 105 | var ( |
| 106 | errSwapOff = errors.New("swap: off on this box") |
| 107 | errSwapBusy = errors.New("swap: another generation operation is in flight") |
| 108 | errSwapKind = errors.New("swap: only exec instances overlap; workers and sidecars keep stop-then-start") |
| 109 | errSuccessorExited = errors.New("swap: successor exited before the switch") |
| 110 | errSwapMode = errors.New("swap: unknown mode") |
| 111 | ) |
| 112 | |
| 113 | func abandon(err error) SwapResult { return SwapResult{Outcome: SwapAbandoned, Reason: err.Error()} } |
| 114 | func failSwap(err error) SwapResult { return SwapResult{Outcome: SwapFailed, Reason: err.Error()} } |
| 115 | |
| 116 | // stepOutcome is how one step of an attempt (a spawn, a health check, a |
| 117 | // readiness or settle window, a pinned wake) ended. |
| 118 | type stepOutcome int |
| 119 | |
| 120 | const ( |
| 121 | stepOK stepOutcome = iota |
| 122 | stepCancelled // the attempt's lifecycle ended it: its context, the barrier, a fence, an intentional stop |
| 123 | stepStale // the attempt gate refused it: a newer attempt, a halted cohort, a lost lease, shutdown |
| 124 | stepFailed // the build failed: spawn, health or readiness |
| 125 | ) |
| 126 | |
| 127 | // classifyStep is THE outcome classifier (spec §6.4, §7.4): every path |
| 128 | // that might record a build failure asks it first, the swaps here and plan |
| 129 | // 5's planned wakes alike. Only stepFailed is a failure of the build; a |
| 130 | // cancelled or stale step records nothing and falls back to nothing. |
| 131 | // lifecycle is the attempt's own context (nil when the caller has none). A |
| 132 | // health deadline that expires is the build's failure (DeadlineExceeded); |
| 133 | // the lifecycle's own end, or a cancellation, is not. |
| 134 | func classifyStep(lifecycle context.Context, err error) stepOutcome { |
| 135 | switch { |
| 136 | case err == nil: |
| 137 | return stepOK |
| 138 | case lifecycle != nil && lifecycle.Err() != nil, |
| 139 | errors.Is(err, context.Canceled), errors.Is(err, errSwapStopped), errors.Is(err, ErrFenced): |
| 140 | return stepCancelled |
| 141 | case verdictOf(err) != gateProceed: |
| 142 | // The attempt gate's own refusal, wherever it surfaced: at Step 3a, |
| 143 | // or as a claims validation inside the work at the exec. Nothing of |
| 144 | // the build ran because of it (spec §7.4). |
| 145 | return stepStale |
| 146 | case errors.Is(err, ErrBuildNotPublished), errors.Is(err, errStagedBuildNotPublished), |
| 147 | errors.Is(err, errBuildStateUnwritten), errors.Is(err, errStaleLink): |
| 148 | // A spawn refused because its build could not be published before |
| 149 | // the exec (ruling R19 item 3; the worker and the staged instance |
| 150 | // sentinels, final review F9): nothing of the build ran, so it is |
| 151 | // abandonment, never a failure of the build. Classified here and not |
| 152 | // in verdictOf, which main's flag-off respawn reads; every caller of |
| 153 | // classifyStep is flag-on (swaps, holds, planned wakes). |
| 154 | // errBuildStateUnwritten (settings spec §4.2) is the same kind of |
| 155 | // refusal; verdictOf also names it, so it is stale through either |
| 156 | // arm, and named here beside its kin. errStaleLink (settings spec |
| 157 | // §4.3 rule 3) is a held wake refusing, before its record and its |
| 158 | // exec, an identity that is not its target: a planned wake's boot |
| 159 | // never ran either. |
| 160 | return stepStale |
| 161 | } |
| 162 | return stepFailed |
| 163 | } |
| 164 | |
| 165 | // gateVerdict is the attempt gate's answer, typed (spec §6.3, §7.4). |
| 166 | type gateVerdict int |
| 167 | |
| 168 | const ( |
| 169 | // gateProceed: the attempt is current and owns the host. |
| 170 | gateProceed gateVerdict = iota |
| 171 | // gateSuperseded: still the owner, but a newer attempt or a halted |
| 172 | // cohort replaced this one. A restore of what it replaced may run, and |
| 173 | // only behind a fresh ownership check at that exec. |
| 174 | gateSuperseded |
| 175 | // gateNoOwnership: the entry is not current, the lease is lost, a fence |
| 176 | // or the agent's shutdown holds the host, or the claims validation |
| 177 | // refused (a changed stamp, a durable fence, a changed claim). Nothing |
| 178 | // may spawn, a restore included. |
| 179 | gateNoOwnership |
| 180 | ) |
| 181 | |
| 182 | // verdictOf types an attempt-gate error. An error that is not the gate's |
| 183 | // (a failure of the build) is gateProceed: the gate did not refuse. |
| 184 | // |
| 185 | // Besides the gate's own refusals, eight kinds of refusal that stop a spawn |
| 186 | // before anything of the build runs are NoOwnership, so the one classifier |
| 187 | // reads them as abandonment, never as a failed build: |
| 188 | // - plan 2's strict-host gate on placed spawns (index P2-8): |
| 189 | // ErrStrictRequired, ErrStrictGroupInUse, ErrStrictCleanupPending; |
| 190 | // - a closed claims-on launch (index P2-9): ErrClaimIsolationUnavailable. |
| 191 | // It wraps hostbox.ErrUnsupported, which is not otherwise classified, |
| 192 | // so it is named here; |
| 193 | // - a row that cannot run a staged generation at all: errUnitNoGens, |
| 194 | // errSwapNoGens; |
| 195 | // - the cgroup manager refusing a generation start because the host's |
| 196 | // parent cgroup holds a process directly (cgroup.ErrLegacyParentPopulated, |
| 197 | // amadan #272): it checks before it creates the child or execs, and the |
| 198 | // condition is the box's, not the build's; |
| 199 | // - the cgroup manager refusing a start because the cgroup it would spawn |
| 200 | // into took cgroup.kill and could not be made fresh |
| 201 | // (cgroup.ErrKilledCgroupKept, amadan #272, Task 19): it refuses before |
| 202 | // it builds the command, so nothing ran, and the condition is the box's; |
| 203 | // - the cgroup manager refusing a generation start because a limit it |
| 204 | // would run under could not be applied, or it could only be started |
| 205 | // outside its cgroup and moved after (cgroup.ErrLimitNotApplied, zdd |
| 206 | // readiness round 2 Task 7): it refuses before it builds the command |
| 207 | // (or, for a failed placement, the shim proved it never reached exec), |
| 208 | // so nothing ran, and the condition is the box's; |
| 209 | // - root refusing the nested staged executable (instancectl.ErrStagedExe, |
| 210 | // Task 9): it validates before it mounts or starts anything; |
| 211 | // - a build state that could not be written before the exec |
| 212 | // (errBuildStateUnwritten, settings spec §4.2). Unlike |
| 213 | // ErrBuildNotPublished it is a new sentinel, so naming it here changes |
| 214 | // nothing main did: a worker respawn it refuses logs a refusal, not |
| 215 | // "failed to respawn" and a member error event, and the sweep asks |
| 216 | // again; a ReplaceWorker attempt it refuses is abandoned and restores |
| 217 | // the build it replaced at once (its veto switch names this sentinel |
| 218 | // before the NoOwnership case), since nothing of the attempt ran. |
| 219 | // |
| 220 | // errSwapStopped (the barrier's cancel, from memberCurrentLocked) is |
| 221 | // NoOwnership too: the entry is stopping, so nothing may spawn for it. |
| 222 | // classifyStep reads it as stepCancelled before it asks this. |
| 223 | func verdictOf(err error) gateVerdict { |
| 224 | switch { |
| 225 | case err == nil: |
| 226 | return gateProceed |
| 227 | case errors.Is(err, errSwapSuperseded): |
| 228 | return gateSuperseded |
| 229 | case errors.Is(err, errSwapNotAwake), errors.Is(err, errSwapLeaseLost), errors.Is(err, errSwapFenced), |
| 230 | errors.Is(err, errSwapShutdown), errors.Is(err, errSwapStopped), errors.Is(err, ErrFenced), |
| 231 | errors.Is(err, hostguard.ErrStampMismatch), errors.Is(err, hostguard.ErrFenced), |
| 232 | errors.Is(err, hostguard.ErrInactive), errors.Is(err, hostguard.ErrRetiredStamp), |
| 233 | errors.Is(err, hostguard.ErrOldStamp), errors.Is(err, hostguard.ErrClaimIncarnation), |
| 234 | errors.Is(err, hostguard.ErrRecordIncarnation), errors.Is(err, hostguard.ErrNotParticipant), |
| 235 | errors.Is(err, hostguard.ErrNoClaim), errors.Is(err, hostguard.ErrNoClaims), |
| 236 | errors.Is(err, hostguard.ErrNoHolder), errors.Is(err, hostguard.ErrMalformedClaim), |
| 237 | errors.Is(err, hostbox.ErrChanged), errors.Is(err, errClaimsAdmission), |
| 238 | errors.Is(err, ErrStrictRequired), errors.Is(err, ErrStrictGroupInUse), errors.Is(err, ErrStrictCleanupPending), |
| 239 | errors.Is(err, ErrClaimIsolationUnavailable), errors.Is(err, cgroup.ErrLegacyParentPopulated), |
| 240 | errors.Is(err, cgroup.ErrKilledCgroupKept), errors.Is(err, cgroup.ErrLimitNotApplied), |
| 241 | errors.Is(err, errUnitNoGens), errors.Is(err, errSwapNoGens), errors.Is(err, instancectl.ErrStagedExe), |
| 242 | errors.Is(err, errBuildStateUnwritten): |
| 243 | return gateNoOwnership |
| 244 | } |
| 245 | return gateProceed |
| 246 | } |
| 247 | |
| 248 | // errClaimsAdmission marks an error a claims-on spawn met before its exec |
| 249 | // hook ran (claimAdmission): the claim read failed (a store error, a |
| 250 | // timeout), the claim was absent or malformed, or the work's validation |
| 251 | // refused before the spawn. Nothing of the build ran, so it is never a |
| 252 | // failure of the build (spec §7.4). |
| 253 | var errClaimsAdmission = errors.New("the claims admission refused before the exec") |
| 254 | |
| 255 | // claimAdmissionPrefix is the text claimAdmission puts before its cause's. |
| 256 | // memberSafeWakeReason removes it, so a member's event reads as the cause. |
| 257 | var claimAdmissionPrefix = errClaimsAdmission.Error() + ": " |
| 258 | |
| 259 | // claimAdmission wraps err in errClaimsAdmission when the spawn never |
| 260 | // reached its exec hook (reached false). An error after the hook ran is the |
| 261 | // spawn's own, and is returned as it is. |
| 262 | func claimAdmission(err error, reached bool) error { |
| 263 | if err == nil || reached { |
| 264 | return err |
| 265 | } |
| 266 | return fmt.Errorf("%w: %w", errClaimsAdmission, err) |
| 267 | } |
| 268 | |
| 269 | // stepResult is classifyStep for a swap step: a cancelled or stale step |
| 270 | // abandons the attempt (over is true); a failure of the build is the |
| 271 | // caller's to report (over is false). |
| 272 | func stepResult(lifecycle context.Context, err error) (SwapResult, bool) { |
| 273 | switch classifyStep(lifecycle, err) { |
| 274 | case stepCancelled: |
| 275 | return abandon(errSwapStopped), true |
| 276 | case stepStale: |
| 277 | return abandon(err), true |
| 278 | } |
| 279 | return SwapResult{}, false |
| 280 | } |
| 281 | |
| 282 | // Swap runs one attempt on an awake entry (spec §7.3–§7.5, §7.8). It |
| 283 | // blocks until the attempt is done, failed or abandoned. For an overlap |
| 284 | // that includes the retiring generation's drain. For a hold it lasts until |
| 285 | // the new generation is healthy or has failed. |
| 286 | // |
| 287 | // newest reports the scheduler's newest revision for a host. It is called |
| 288 | // with a.mu held, so it must not block and must not call back into the |
| 289 | // Service. |
| 290 | func (a *Service) Swap(ctx context.Context, at Attempt, newest func(host string) uint64) SwapResult { |
| 291 | if !a.gens { |
| 292 | return abandon(errSwapOff) |
| 293 | } |
| 294 | gb, ok := a.backend.(SwapBackend) |
| 295 | if !ok { |
| 296 | return abandon(errSwapNoGens) |
| 297 | } |
| 298 | if newest == nil { |
| 299 | return abandon(errSwapSuperseded) |
| 300 | } |
| 301 | switch at.Mode { |
| 302 | case ModeOverlap: |
| 303 | if a.foreignRow(at.Host) { |
| 304 | // A foreign row keeps its canonical socket (plan 1), so two |
| 305 | // generations of it cannot coexist: its replacement is a hold |
| 306 | // (spec §2.9, rev 17). Its result reads as a hold's, SwapDown |
| 307 | // when the new build fails, and Mode tells the scheduler so: a |
| 308 | // hold build that ran is held down (spec §6.4, §8.3). |
| 309 | res := a.swapHold(ctx, at, newest) |
| 310 | res.Mode = ModeHold |
| 311 | return res |
| 312 | } |
| 313 | res := a.swapOverlap(ctx, gb, at, newest) |
| 314 | res.Mode = ModeOverlap |
| 315 | return res |
| 316 | case ModeHold: |
| 317 | res := a.swapHold(ctx, at, newest) |
| 318 | res.Mode = ModeHold |
| 319 | return res |
| 320 | } |
| 321 | return abandon(errSwapMode) |
| 322 | } |
| 323 | |
| 324 | // EffectiveMode is the mode Swap runs an attempt of mode m on host in: an |
| 325 | // overlap on a foreign-layout row runs as a hold (foreignRow, spec §2.9), |
| 326 | // anything else as given. The scheduler writes it into the attempt's |
| 327 | // write-ahead intent before calling Swap (rollout Task 5, plan review |
| 328 | // finding 2), so an agent that crashes mid-swap reopens its failure book |
| 329 | // with the hold it was running. Called without a.mu held. |
| 330 | func (a *Service) EffectiveMode(host string, m SwapMode) SwapMode { |
| 331 | if m == ModeOverlap && a.foreignRow(host) { |
| 332 | return ModeHold |
| 333 | } |
| 334 | return m |
| 335 | } |
| 336 | |
| 337 | // attemptCtx is the attempt's own lifetime. It ends with the caller's ctx, |
| 338 | // with the agent (Shutdown cancels lifeCtx), or with the barrier, which |
| 339 | // calls the returned cancel through the member. |
| 340 | func (a *Service) attemptCtx(ctx context.Context) (context.Context, context.CancelFunc) { |
| 341 | mctx, cancel := context.WithCancel(ctx) |
| 342 | if a.lifeCtx == nil { |
| 343 | return mctx, cancel |
| 344 | } |
| 345 | stop := context.AfterFunc(a.lifeCtx, cancel) |
| 346 | return mctx, func() { stop(); cancel() } |
| 347 | } |
| 348 | |
| 349 | // genHealthy is the successor's health check, through its own handle: |
| 350 | // GET / below 500 within spawnHealthTimeout (spec §7.5 step 4). |
| 351 | func (a *Service) genHealthy(ctx context.Context, g Gen) error { |
| 352 | if a.genHealth != nil { |
| 353 | return a.genHealth(ctx, g) |
| 354 | } |
| 355 | if g.Socket == "" { |
| 356 | return errors.New("generation has no socket") |
| 357 | } |
| 358 | return waitHealthyCtx(ctx, g.Socket, spawnHealthTimeout) |
| 359 | } |
| 360 | |
| 361 | // spawnSuccessor is §7.5 step 3 in both modes. Claims off, it is the |
| 362 | // backend's staged spawn. Claims on, a claim-backed spawn would run inside |
| 363 | // the selected work, which validates the claim and stamp at the exec. |
| 364 | // |
| 365 | // gate is the attempt gate (memberCurrentLocked, taking a.mu itself). It |
| 366 | // runs immediately before every exec: claims off, right before |
| 367 | // StartStaged, and again inside it at the backend's final launch point, |
| 368 | // after the backend's preparation (withExecGate; final review F1). Claims |
| 369 | // on, it belongs after the runtime's Read and Prepare |
| 370 | // and again inside the selected work at the exec, as the spawn's before |
| 371 | // hook (plan 3 Task 10). |
| 372 | // |
| 373 | // Every claims-on launch is closed on this base (index P2-9): the isolated |
| 374 | // root launcher cannot carry claim reservations. swapOverlap refuses every |
| 375 | // claims-backed attempt before its raise (claimsSwapRefusal), so production |
| 376 | // never reaches the claims-on arm here; only the claimSpawn test seam, the |
| 377 | // stand-in for an open launch, does. Without the seam the arm still refuses, |
| 378 | // as a claims admission (NoOwnership, the attempt abandoned), and a |
| 379 | // claims-on spawn is never sent down the claims-off StartStaged. |
| 380 | func (a *Service) spawnSuccessor(ctx context.Context, gb SwapBackend, e *actEntry, rt registry.Route, work hostwrite.Work, g Gen, gate func() error) (Gen, Proc, error) { |
| 381 | if hostwrite.Legacy(work) { |
| 382 | if err := gate(); err != nil { |
| 383 | return g, nil, err |
| 384 | } |
| 385 | // Settings spec §4.2: the successor's build is recorded before it |
| 386 | // runs against the data, under the incumbent's lease. A refusal is |
| 387 | // abandonment (classifyStep), never the build's failure. |
| 388 | if err := a.recordBuild(ctx, rt.Host, g.Build); err != nil { |
| 389 | return g, nil, err |
| 390 | } |
| 391 | return gb.StartStaged(withExecGate(ctx, gate), rt, g) |
| 392 | } |
| 393 | if a.claimSpawn != nil { |
| 394 | return a.claimSpawn(ctx, e, rt, work, g) |
| 395 | } |
| 396 | return g, nil, claimAdmission(ErrClaimIsolationUnavailable, false) |
| 397 | } |
| 398 | |
| 399 | // swapOverlap is §7.5 steps 1 to 7. The successor is registered in the |
| 400 | // operation set under the same lock that runs the §7.4 checks, so a |
| 401 | // terminal stop that begins later always sees it and joins it. |
| 402 | // |
| 403 | // Every exit after the admission leaves the set through dropMember (or, on |
| 404 | // success, promotes the successor and drops the retiring generation), and |
| 405 | // leaveSet restores a raised parent when the last member leaves. Nothing |
| 406 | // here blocks under a.mu except GenSwitched, which plan 1's lock contract |
| 407 | // puts in the same critical section as e.gen. |
| 408 | func (a *Service) swapOverlap(ctx context.Context, gb SwapBackend, at Attempt, newest func(string) uint64) (out SwapResult) { |
| 409 | // The execution evidence (SwapResult.Executed): the backend returned a |
| 410 | // successor process, or its error says the build may have run |
| 411 | // (startMayHaveRun), whatever ends the attempt after that. |
| 412 | var executed bool |
| 413 | defer func() { out.Executed = executed }() |
| 414 | // The bytes first. An artifact that is not what the attempt chose never |
| 415 | // reaches admission, let alone exec (spec §4.2, §7.5 step 3). |
| 416 | if err := verifyStaged(at.StagedPath, at.Digest); err != nil { |
| 417 | return abandon(fmt.Errorf("staging: %w", err)) |
| 418 | } |
| 419 | work, err := a.selectWork(ctx, at.Host) |
| 420 | if err != nil { |
| 421 | return abandon(fmt.Errorf("admission: %w", err)) |
| 422 | } |
| 423 | if !hostwrite.Legacy(work) { |
| 424 | // Claims on: refused before the admission and the raise, whatever |
| 425 | // the runtime's capabilities (claimsSwapRefusal). |
| 426 | if err := a.claimsSwapRefusal(true); err != nil { |
| 427 | return abandon(err) |
| 428 | } |
| 429 | } |
| 430 | mctx, cancel := a.attemptCtx(ctx) |
| 431 | defer cancel() |
| 432 | m := &genMember{ |
| 433 | gen: Gen{N: nextGen(), Build: at.Build, StagedPath: at.StagedPath, Digest: at.Digest, Demand: at.Demand}, |
| 434 | role: roleSpawning, |
| 435 | cancel: cancel, |
| 436 | done: make(chan struct{}), |
| 437 | } |
| 438 | var ( |
| 439 | e *actEntry |
| 440 | rt registry.Route |
| 441 | incumbent Gen // the current generation at admission, for its tier |
| 442 | ) |
| 443 | // Step 1: the checks and joining the set, inside the selected work. Only |
| 444 | // the registration runs there. A claims-on spawn re-enters the work |
| 445 | // itself (Task 10), and hostguard's flock does not nest. Registering |
| 446 | // under the same lock as the checks is what lets a stop that begins |
| 447 | // afterwards see the successor and join it. |
| 448 | err = a.admit(ctx, work, func() error { |
| 449 | a.mu.Lock() |
| 450 | defer a.mu.Unlock() |
| 451 | e = a.entries[at.Host] |
| 452 | if err := a.swapChecksLocked(e, at, newest); err != nil { |
| 453 | return err |
| 454 | } |
| 455 | if e.worker || e.route.Kind != "instance" { |
| 456 | return errSwapKind |
| 457 | } |
| 458 | // A foreign row keeps its canonical socket (plan 1), so two |
| 459 | // generations of it cannot coexist. It may still hold-swap. So |
| 460 | // can a flat row whose generation path would not fit (rev 19 |
| 461 | // rule 4), which swapLayout reads as foreign. |
| 462 | if l, ok := a.swapLayout(e.route); ok && l != LayoutFlat && l != LayoutNested { |
| 463 | return errSwapLayout |
| 464 | } |
| 465 | // A parent still marked raised with no member is a restore that |
| 466 | // failed and awaits the sweep's retry (final review F11): an overlap |
| 467 | // raising it again now could have that retry lower it under the |
| 468 | // successor. |
| 469 | if len(e.ops.members) > 0 || e.ops.raised { |
| 470 | return errSwapBusy |
| 471 | } |
| 472 | // A run's unit may be starting against the incumbent right now |
| 473 | // (merge ruling M2). Admitting now could retire the incumbent under |
| 474 | // a command that has not even started; the attempt is abandoned |
| 475 | // instead, and the scheduler tries again once the start has |
| 476 | // returned. From here on RunCmd waits for this successor rather |
| 477 | // than starting a unit (switchPendingLocked). |
| 478 | if cmdStartInFlightLocked(e) { |
| 479 | return errSwapBusy |
| 480 | } |
| 481 | a.addMemberLocked(e, m) |
| 482 | rt, incumbent = e.route, e.gen |
| 483 | return nil |
| 484 | }) |
| 485 | if err != nil { |
| 486 | // The work may refuse after its body registered the member. |
| 487 | if a.isMember(e, m) { |
| 488 | a.dropMember(e, m) |
| 489 | } |
| 490 | return abandon(err) |
| 491 | } |
| 492 | |
| 493 | // Each artifact's own tier, captured now (spec §7.3): the incumbent's |
| 494 | // from the build it runs, the successor's from the attempt's build. |
| 495 | // Neither is read from the live link, which root may repoint. They size |
| 496 | // the overlap parent, the successor's child, and the restore (leaveSet |
| 497 | // restores to the current generation's Tier). |
| 498 | if incumbent.Tier == (cgroup.Limit{}) { |
| 499 | incumbent.Tier = gb.GenTier(rt, incumbent.Build) |
| 500 | } |
| 501 | succTier := gb.GenTier(rt, at.Build) |
| 502 | a.mu.Lock() |
| 503 | if e.gen.N == incumbent.N { |
| 504 | e.gen.Tier = incumbent.Tier |
| 505 | } |
| 506 | m.gen.Tier = succTier |
| 507 | a.mu.Unlock() |
| 508 | |
| 509 | // Step 2: the parent raise, before the successor spawns (spec §7.3). |
| 510 | // The parent is marked potentially changed BEFORE the raise is asked: a |
| 511 | // raise that wrote memory.max and then failed a later write has changed |
| 512 | // it, and the member's cleanup (leaveSet) must restore it (final review |
| 513 | // F11). |
| 514 | a.mu.Lock() |
| 515 | e.ops.raised = true |
| 516 | a.mu.Unlock() |
| 517 | if err := gb.RaiseGenParent(ctx, rt, incumbent.Tier, succTier); err != nil { |
| 518 | if parentUntouched(err) { |
| 519 | // Refused before any write: there is nothing to restore. |
| 520 | a.mu.Lock() |
| 521 | e.ops.raised = false |
| 522 | a.mu.Unlock() |
| 523 | } |
| 524 | a.dropMember(e, m) // restores a possibly changed parent (leaveSet) |
| 525 | return abandon(fmt.Errorf("raise parent limit: %w", err)) |
| 526 | } |
| 527 | |
| 528 | // The successor's build is published before its exec (spec §4.2, §8.5): |
| 529 | // it has been in the operation set since admission, so this write names |
| 530 | // it. A failed write abandons the attempt; nothing of it ran, so it is |
| 531 | // never a failure of the build. |
| 532 | if err := a.publishBeforeExec(); err != nil { |
| 533 | a.dropMember(e, m) // restores the raised parent (leaveSet) |
| 534 | return abandon(err) |
| 535 | } |
| 536 | |
| 537 | // Step 3a: the §7.4 checks again, at the exec (spec §6.3, §7.4). The |
| 538 | // raise (and the pin publication just above this step) took time: |
| 539 | // a newer attempt, a halted cohort, a lost lease or a fence since |
| 540 | // admission means nothing execs. Claims on, this runs inside the |
| 541 | // selected work as the admission did. |
| 542 | gate := func() error { |
| 543 | a.mu.Lock() |
| 544 | defer a.mu.Unlock() |
| 545 | return a.memberCurrentLocked(e, m, mctx, at, newest) |
| 546 | } |
| 547 | if err := a.admit(ctx, work, gate); err != nil { |
| 548 | a.dropMember(e, m) // restores the raised parent (leaveSet) |
| 549 | return abandon(err) |
| 550 | } |
| 551 | |
| 552 | // Step 3: exec the attempt's staged build by path. The backend reports |
| 553 | // what it ran, and anything but the attempt's choice is refused. |
| 554 | g, proc, spawnErr := a.spawnSuccessor(mctx, gb, e, rt, work, m.gen, gate) |
| 555 | executed = startMayHaveRun(proc, spawnErr) |
| 556 | if spawnErr == nil && (g.N != m.gen.N || g.Build != m.gen.Build || g.StagedPath != m.gen.StagedPath) { |
| 557 | spawnErr = fmt.Errorf("backend ran generation %d of %s from %q; the attempt chose %d of %s from %q", |
| 558 | g.N, g.Build, g.StagedPath, m.gen.N, m.gen.Build, m.gen.StagedPath) |
| 559 | } |
| 560 | a.mu.Lock() |
| 561 | m.proc = proc |
| 562 | if spawnErr == nil { |
| 563 | m.gen, m.role = g, roleSuccessor |
| 564 | } |
| 565 | a.mu.Unlock() |
| 566 | if spawnErr != nil { |
| 567 | a.dropMember(e, m) |
| 568 | if res, over := stepResult(mctx, spawnErr); over { |
| 569 | return res |
| 570 | } |
| 571 | return failSwap(fmt.Errorf("spawn: %w", spawnErr)) |
| 572 | } |
| 573 | |
| 574 | // Step 4: the health check, through the generation's own handle. |
| 575 | if err := a.genHealthy(mctx, m.gen); err != nil { |
| 576 | a.dropMember(e, m) |
| 577 | if res, over := stepResult(mctx, err); over { |
| 578 | return res |
| 579 | } |
| 580 | return failSwap(fmt.Errorf("health: %w", err)) |
| 581 | } |
| 582 | |
| 583 | // Step 4, the canary's half: the 10 s settle (spec §6.2). An |
| 584 | // intentional stop of the entry cancels the observation through the |
| 585 | // barrier, and that is not a failure (spec §6.4). |
| 586 | if at.Canary { |
| 587 | switch observeSettle(mctx, m.proc, swapSettle) { |
| 588 | case settleCancelled: |
| 589 | a.dropMember(e, m) |
| 590 | return abandon(errSwapStopped) |
| 591 | case settleExited: |
| 592 | a.dropMember(e, m) |
| 593 | return failSwap(errSettleExit) |
| 594 | } |
| 595 | } |
| 596 | |
| 597 | // Step 5: the switch. The checks run again, then, under the entry lock, |
| 598 | // AppSocket, the serving build and the cache generation all move with |
| 599 | // e.gen (contract C2). |
| 600 | var ( |
| 601 | retiring *genMember |
| 602 | switched bool |
| 603 | ) |
| 604 | // The host's schedule write fence (final review F8): the switch waits |
| 605 | // for the schedule writes in flight, keeps new ones out, and latches the |
| 606 | // retiring generation in the same step, so no write of it lands after |
| 607 | // the switch. Taken before a.mu (the fence's lock order). |
| 608 | fence := a.schedFenceFor(at.Host) |
| 609 | err = a.admit(ctx, work, func() error { |
| 610 | fence.lock() |
| 611 | defer fence.unlock() |
| 612 | a.mu.Lock() |
| 613 | defer a.mu.Unlock() |
| 614 | if err := a.memberCurrentLocked(e, m, mctx, at, newest); err != nil { |
| 615 | return err |
| 616 | } |
| 617 | if m.proc == nil || m.proc.Exited() { |
| 618 | return errSuccessorExited |
| 619 | } |
| 620 | // Admission refused a run's start in flight and RunCmd starts none |
| 621 | // while this successor is live, so none can be in flight here |
| 622 | // (merge ruling M2). Checked again because a switch over one would |
| 623 | // retire the generation that run is starting against. |
| 624 | if cmdStartInFlightLocked(e) { |
| 625 | return errSwapBusy |
| 626 | } |
| 627 | // The canonical symlink, the purge and the socket mode first. If the |
| 628 | // link cannot move, nothing has switched yet. This runs under a.mu, |
| 629 | // in the same critical section that sets e.gen below: plan 1's lock |
| 630 | // contract for swapCanonicalLink. |
| 631 | if err := gb.GenSwitched(e.route, m.gen, m.proc); err != nil { |
| 632 | return err |
| 633 | } |
| 634 | retiring = &genMember{gen: e.gen, role: roleRetiring, proc: e.proc, done: make(chan struct{})} |
| 635 | fence.retire(retiring.gen.N) // its schedule writes are refused from here, for good |
| 636 | delete(e.ops.members, m.gen.N) |
| 637 | a.addMemberLocked(e, retiring) // swapsim:M2 |
| 638 | e.gen, e.proc = m.gen, m.proc |
| 639 | e.spawnVersion, e.staleSince, e.spawnedAt = m.gen.Build, time.Time{}, a.now() |
| 640 | a.noteCurrent(e.host, m.gen.N) // plan 1's cache generation (C4) |
| 641 | // Runs admitted against the retiring generation are interrupted |
| 642 | // with it, as a scheduled restart's stop interrupts them (merge |
| 643 | // ruling M2): a command never outlives the build it was started |
| 644 | // from as that build retires. |
| 645 | a.interruptCmdsLocked(e, retiring.gen.N, stopWhyFor(at.Cause)) |
| 646 | switched = true |
| 647 | return nil |
| 648 | }) |
| 649 | if err != nil && switched { |
| 650 | // The work refused after its body had switched. The successor is |
| 651 | // current and serving, so it is never dropped: the swap finishes, |
| 652 | // and the retiring generation still drains below. |
| 653 | log.Printf("activator: %s: the selected work reported %v after the switch to generation %d; the switch stands", at.Host, err, m.gen.N) |
| 654 | } else if err != nil { |
| 655 | a.dropMember(e, m) |
| 656 | if errors.Is(err, errSuccessorExited) { |
| 657 | return failSwap(err) |
| 658 | } |
| 659 | return abandon(err) |
| 660 | } |
| 661 | close(m.done) // promoted: it is current now, no longer a member |
| 662 | // The retiring generation is no longer current. The edge closes its idle |
| 663 | // connections now (spec §7.2), outside every activator lock (C4). |
| 664 | a.genRetired(at.Host, retiring.gen) |
| 665 | |
| 666 | // Steps 6 and 7. The retiring generation drains under its own SIGTERM |
| 667 | // handling, bounded by actStopTimeout. Once it is proven empty, |
| 668 | // leaveSet restores the parent limit. The good build (§8.5) is plan 5's, |
| 669 | // recorded from this result. |
| 670 | a.mu.Lock() |
| 671 | rt = e.route |
| 672 | a.mu.Unlock() |
| 673 | emitEvent(a.Logs, rt, "info", "instance stopping") |
| 674 | res := SwapResult{Outcome: SwapDone, Gen: m.gen} |
| 675 | // The drain is timed on the wall clock (not a.now, which tests fake), for |
| 676 | // the "swapped" event's drain time (spec §9.4). |
| 677 | drainStart := time.Now() |
| 678 | proven := a.dropMember(e, retiring) |
| 679 | res.Drain = time.Since(drainStart) |
| 680 | if !proven { |
| 681 | res.Reason = "retiring generation not yet proven stopped; the sweep finishes it" |
| 682 | } |
| 683 | // Step 8: a legacy co-spawned sidecar moves onto the new generation. |
| 684 | a.replaceSidecar(e) |
| 685 | return res |
| 686 | } |
| 687 | |
amadan