| 1 | package activator |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "log" |
| 7 | "net/http" |
| 8 | "time" |
| 9 | |
| 10 | "github.com/carlosframework/platform/internal/controlname" |
| 11 | "github.com/carlosframework/platform/internal/edge" |
| 12 | "github.com/carlosframework/platform/internal/registry" |
| 13 | ) |
| 14 | |
| 15 | var ( |
| 16 | ErrFenceNotInstalled = errors.New("activator: maintenance fence not installed") |
| 17 | ErrFenced = errors.New("activator: maintenance fence active") |
| 18 | ErrFenceGeneration = errors.New("activator: maintenance generation refused") |
| 19 | ErrStopUnproven = errors.New("activator: stop evidence unavailable") |
| 20 | ErrFenceHeld = errors.New("activator: another maintenance fence is held") |
| 21 | ) |
| 22 | |
| 23 | // StopEvidence records local process and replica closure. It grants no lease, |
| 24 | // cgroup, database or resume authority. |
| 25 | type StopEvidence struct { |
| 26 | host string |
| 27 | generation uint64 |
| 28 | observation *stopObservation |
| 29 | } |
| 30 | |
| 31 | func (e *StopEvidence) Matches(host string, generation uint64) bool { |
| 32 | return e != nil && e.host == host && e.generation == generation && e.observation != nil && e.observation.complete() |
| 33 | } |
| 34 | |
| 35 | type stopObservation struct { |
| 36 | leaseReleased bool |
| 37 | mainStop, mainExited bool |
| 38 | sidecarStop, sidecarExited bool |
| 39 | replicaClosed bool |
| 40 | uncertain bool |
| 41 | // owedBy is the one retained failed wake whose unproven legacy start |
| 42 | // (#292) is this observation's only uncertainty, or nil. Only that |
| 43 | // wake's proof discharges it (reapUnprovenWake); any other source of |
| 44 | // uncertainty clears it, and then the uncertainty stays. |
| 45 | owedBy *unprovenWake |
| 46 | // noStart: a wake refused before anything of it started (notStarted). |
| 47 | // It is evidence for a host with no stop history, and leaves any |
| 48 | // history the host has exactly as it was (rememberStopLocked). |
| 49 | noStart bool |
| 50 | } |
| 51 | |
| 52 | func (o *stopObservation) complete() bool { |
| 53 | return o != nil && !o.uncertain && o.mainStop && o.mainExited && o.sidecarStop && o.sidecarExited && o.replicaClosed |
| 54 | } |
| 55 | |
| 56 | // wakeStartedNothing reports whether a failed wake holds no process, |
| 57 | // replica or sidecar of its own: the condition for notStarted. |
| 58 | func wakeStartedNothing(w *wokenInstance) bool { |
| 59 | return w != nil && w.proc == nil && w.rep == nil && w.sidecar == nil |
| 60 | } |
| 61 | |
| 62 | // notStarted records a wake whose launch is known not to have started: no |
| 63 | // process, no sidecar and no replica of it exist, so every stage holds and |
| 64 | // nothing is uncertain (seeded pre-land review, P2). |
| 65 | func (o *stopObservation) notStarted() { |
| 66 | o.mainStop, o.mainExited = true, true |
| 67 | o.sidecarStop, o.sidecarExited = true, true |
| 68 | o.replicaClosed = true |
| 69 | o.uncertain = false |
| 70 | o.noStart = true |
| 71 | } |
| 72 | |
| 73 | // FenceAndStop installs a local fence before collecting stop evidence. Once this |
| 74 | // call installs its fence, cancellation or failure leaves admission closed. |
| 75 | // ErrFenceNotInstalled means this call could not install or confirm its fence; |
| 76 | // it makes no claim about admission, including a fence from an earlier call. |
| 77 | // ErrStopUnproven guarantees this call installed a fence, but proves no stop. |
| 78 | // A missing entry is not evidence that a prior process or replica stopped. |
| 79 | func (a *Service) FenceAndStop(ctx context.Context, host string, generation uint64) (*StopEvidence, error) { |
| 80 | return a.fenceAndStop(ctx, host, generation, false, "") |
| 81 | } |
| 82 | |
| 83 | // FenceAndStopIfUnfenced is FenceAndStop for a caller that must never take |
| 84 | // over somebody else's fence. FenceAndStop replaces a fence at a lower |
| 85 | // generation, and a caller that then lifts its own generation reopens |
| 86 | // admission the other operation closed. Here the check and the install |
| 87 | // happen under the same lock: if any fence is held for host, of any |
| 88 | // generation, it answers ErrFenceHeld, installs nothing and stops nothing. |
| 89 | // Otherwise it is FenceAndStop, answers included. |
| 90 | func (a *Service) FenceAndStopIfUnfenced(ctx context.Context, host string, generation uint64) (*StopEvidence, error) { |
| 91 | return a.fenceAndStop(ctx, host, generation, true, "") |
| 92 | } |
| 93 | |
| 94 | // FenceAndStopAs is FenceAndStopIfUnfenced for a caller that must be able |
| 95 | // to meet its own fence again: a retry after a lost answer, or after the |
| 96 | // caller's own later step failed. owner is the caller's persisted name for |
| 97 | // this one fence. Under the lock that installs, a held fence is the |
| 98 | // caller's only when it was installed by FenceAndStopAs with exactly this |
| 99 | // owner at exactly this generation; then the call is FenceAndStop under it |
| 100 | // (it re-collects the stop evidence). Any other held fence, of any |
| 101 | // generation and however installed, answers ErrFenceHeld, installs nothing |
| 102 | // and stops nothing. Fences live only in this process, so after a restart |
| 103 | // no fence is the caller's, whatever it persisted. An empty owner is |
| 104 | // ErrFenceGeneration. |
| 105 | func (a *Service) FenceAndStopAs(ctx context.Context, host string, generation uint64, owner string) (*StopEvidence, error) { |
| 106 | if owner == "" { |
| 107 | return nil, ErrFenceGeneration |
| 108 | } |
| 109 | return a.fenceAndStop(ctx, host, generation, true, owner) |
| 110 | } |
| 111 | |
| 112 | func (a *Service) fenceAndStop(ctx context.Context, host string, generation uint64, exclusive bool, owner string) (*StopEvidence, error) { |
| 113 | if a == nil || !controlname.ValidHostShape(host) || generation == 0 { |
| 114 | return nil, ErrFenceGeneration |
| 115 | } |
| 116 | if ctx == nil { |
| 117 | return nil, ErrFenceNotInstalled |
| 118 | } |
| 119 | deadline, ok := ctx.Deadline() |
| 120 | if !ok || ctx.Err() != nil || !time.Now().Before(deadline) { |
| 121 | return nil, ErrFenceNotInstalled |
| 122 | } |
| 123 | installed := false |
| 124 | for { |
| 125 | if err := a.lockFence(ctx); err != nil { |
| 126 | if installed { |
| 127 | return nil, ErrStopUnproven |
| 128 | } |
| 129 | return nil, ErrFenceNotInstalled |
| 130 | } |
| 131 | if a.seedExclusions[host] != nil { |
| 132 | a.mu.Unlock() |
| 133 | return nil, ErrFenceHeld |
| 134 | } |
| 135 | if preparation := a.tokenPreparations[host]; preparation != nil { |
| 136 | a.mu.Unlock() |
| 137 | select { |
| 138 | case <-ctx.Done(): |
| 139 | if installed { |
| 140 | return nil, ErrStopUnproven |
| 141 | } |
| 142 | return nil, ErrFenceNotInstalled |
| 143 | case <-preparation: |
| 144 | continue |
| 145 | } |
| 146 | } |
| 147 | if a.fences == nil { |
| 148 | a.fences = make(map[string]uint64) |
| 149 | } |
| 150 | prior := a.fences[host] |
| 151 | own := owner != "" && prior == generation && a.fenceOwners[host] == owner |
| 152 | // Only before this call installs: on a later round of the loop the |
| 153 | // fence held is this call's own. |
| 154 | if exclusive && !installed && prior != 0 && !own { |
| 155 | a.mu.Unlock() |
| 156 | return nil, ErrFenceHeld |
| 157 | } |
| 158 | if owner != "" && installed && !own { |
| 159 | // An owned fence replaced while this call waited is no longer |
| 160 | // this call's: it installed one, and proves nothing more. |
| 161 | a.mu.Unlock() |
| 162 | return nil, ErrStopUnproven |
| 163 | } |
| 164 | if prior > generation { |
| 165 | a.mu.Unlock() |
| 166 | return nil, ErrFenceGeneration |
| 167 | } |
| 168 | a.fences[host] = generation |
| 169 | if owner != "" { |
| 170 | if a.fenceOwners == nil { |
| 171 | a.fenceOwners = make(map[string]string) |
| 172 | } |
| 173 | a.fenceOwners[host] = owner |
| 174 | } else { |
| 175 | delete(a.fenceOwners, host) |
| 176 | } |
| 177 | installed = true |
| 178 | e := a.entries[host] |
| 179 | if e == nil { |
| 180 | outcome := a.stopHistory[host] |
| 181 | a.mu.Unlock() |
| 182 | if ctx.Err() != nil || !time.Now().Before(deadline) { |
| 183 | return nil, ErrStopUnproven |
| 184 | } |
| 185 | if !outcome.complete() { |
| 186 | return nil, ErrStopUnproven |
| 187 | } |
| 188 | return &StopEvidence{host: host, generation: generation, observation: outcome}, nil |
| 189 | } |
| 190 | var wait <-chan struct{} |
| 191 | switch e.state { |
| 192 | case actWaking: |
| 193 | if e.wakeCancel != nil { |
| 194 | e.wakeCancel() |
| 195 | } |
| 196 | wait = e.ready |
| 197 | case actAwake: |
| 198 | a.beginStopLocked(e, StopFenced) |
| 199 | wait = e.stopped |
| 200 | case actStopping: |
| 201 | wait = e.stopped |
| 202 | default: |
| 203 | a.mu.Unlock() |
| 204 | return nil, ErrStopUnproven |
| 205 | } |
| 206 | a.mu.Unlock() |
| 207 | select { |
| 208 | case <-ctx.Done(): |
| 209 | return nil, ErrStopUnproven |
| 210 | case <-wait: |
| 211 | } |
| 212 | } |
| 213 | } |
| 214 | |
| 215 | // LiftFence removes the fence FenceAndStop installed at generation, and only |
| 216 | // that one: a fence at any other generation stays, so a caller holding a stale |
| 217 | // generation cannot reopen admission somebody else closed. It reports whether |
| 218 | // it lifted anything. The caller owns the proof that nothing on the far side |
| 219 | // of the fence still needs it closed; the move worker lifts only once this |
| 220 | // box no longer holds the host's route (instance-moves design §6.1, G12). |
| 221 | func (a *Service) LiftFence(host string, generation uint64) bool { |
| 222 | if a == nil || generation == 0 { |
| 223 | return false |
| 224 | } |
| 225 | a.mu.Lock() |
| 226 | defer a.mu.Unlock() |
| 227 | if a.seedExclusions[host] != nil || a.fences[host] != generation { |
| 228 | return false |
| 229 | } |
| 230 | delete(a.fences, host) |
| 231 | delete(a.fenceOwners, host) |
| 232 | return true |
| 233 | } |
| 234 | |
| 235 | // LiftFenceAs is LiftFence for an owned fence: it lifts host's fence only |
| 236 | // when it is exactly the one FenceAndStopAs installed with owner at |
| 237 | // generation, and reports whether it lifted anything. A fence another |
| 238 | // operation installed at the same generation stays. |
| 239 | func (a *Service) LiftFenceAs(host string, generation uint64, owner string) bool { |
| 240 | if a == nil || generation == 0 || owner == "" { |
| 241 | return false |
| 242 | } |
| 243 | a.mu.Lock() |
| 244 | defer a.mu.Unlock() |
| 245 | if a.seedExclusions[host] != nil || a.fences[host] != generation || a.fenceOwners[host] != owner { |
| 246 | return false |
| 247 | } |
| 248 | delete(a.fences, host) |
| 249 | delete(a.fenceOwners, host) |
| 250 | return true |
| 251 | } |
| 252 | |
| 253 | func (a *Service) lockFence(ctx context.Context) error { |
| 254 | timer := time.NewTicker(time.Millisecond) |
| 255 | defer timer.Stop() |
| 256 | for { |
| 257 | if ctx.Err() != nil { |
| 258 | return ErrStopUnproven |
| 259 | } |
| 260 | if a.mu.TryLock() { |
| 261 | if ctx.Err() != nil { |
| 262 | a.mu.Unlock() |
| 263 | return ErrStopUnproven |
| 264 | } |
| 265 | return nil |
| 266 | } |
| 267 | select { |
| 268 | case <-ctx.Done(): |
| 269 | return ErrStopUnproven |
| 270 | case <-timer.C: |
| 271 | } |
| 272 | } |
| 273 | } |
| 274 | func (a *Service) wakeAllowed(host string) bool { |
| 275 | a.mu.Lock() |
| 276 | defer a.mu.Unlock() |
| 277 | return a.fences[host] == 0 |
| 278 | } |
| 279 | |
| 280 | // Fenced reports whether a maintenance fence is installed for host right |
| 281 | // now. It is the layout pass's own affirmative read of the same map |
| 282 | // wakeAllowed checks, so a pass deciding whether it is safe to install its |
| 283 | // own fence can ask the question the way it is phrased in its own logic, |
| 284 | // rather than negating an admission check meant for something else. |
| 285 | func (a *Service) Fenced(host string) bool { |
| 286 | a.mu.Lock() |
| 287 | defer a.mu.Unlock() |
| 288 | return a.fences[host] != 0 |
| 289 | } |
| 290 | |
| 291 | // FenceGeneration is the generation of the fence installed for host right |
| 292 | // now, or 0 when there is none. The layout pass uses it to tell a fence it |
| 293 | // kept itself from another operation's, since both answer ErrFenceHeld to |
| 294 | // its exclusive install. |
| 295 | func (a *Service) FenceGeneration(host string) uint64 { |
| 296 | a.mu.Lock() |
| 297 | defer a.mu.Unlock() |
| 298 | return a.fences[host] |
| 299 | } |
| 300 | |
| 301 | // StopUncertain reports whether host's process state cannot yet be trusted |
| 302 | // as stopped: either an entry for it is still live right now, in any state |
| 303 | // (waking, awake or stopping), or the last recorded stop observation never |
| 304 | // finished every check FenceAndStop requires for proof. A pass that meets |
| 305 | // ErrStopUnproven for an entry-less host still needs this before it can tell |
| 306 | // "nothing was ever running" apart from "something was running and its |
| 307 | // closure was never confirmed"; only the first of those is safe to treat as |
| 308 | // a completed stop. |
| 309 | func (a *Service) StopUncertain(host string) bool { |
| 310 | a.mu.Lock() |
| 311 | defer a.mu.Unlock() |
| 312 | if a.entries[host] != nil { |
| 313 | return true |
| 314 | } |
| 315 | outcome, ok := a.stopHistory[host] |
| 316 | return ok && !outcome.complete() |
| 317 | } |
| 318 | func (a *Service) rememberStopLocked(host string, observation *stopObservation) { |
| 319 | if a.stopHistory == nil { |
| 320 | a.stopHistory = make(map[string]*stopObservation) |
| 321 | } |
| 322 | prior := a.stopHistory[host] |
| 323 | if observation.noStart && prior != nil { |
| 324 | // A wake that started nothing says nothing about what an earlier |
| 325 | // one left: that record, its obligations and their owner, stay. |
| 326 | return |
| 327 | } |
| 328 | if prior != nil && prior.uncertain { |
| 329 | retained := *observation |
| 330 | retained.uncertain = true |
| 331 | // The prior's uncertainty stays one retained wake's alone only |
| 332 | // while that same wake is the one recording. |
| 333 | retained.owedBy = nil |
| 334 | if observation.uncertain && observation.owedBy == prior.owedBy { |
| 335 | retained.owedBy = prior.owedBy |
| 336 | } |
| 337 | observation = &retained |
| 338 | } |
| 339 | a.stopHistory[host] = observation |
| 340 | } |
| 341 | func maintenanceResponse(w http.ResponseWriter, r *http.Request) { |
| 342 | edge.MarkAnswered(r) |
| 343 | w.Header().Set("Cache-Control", "no-store") |
| 344 | w.Header().Set("Retry-After", "5") |
| 345 | http.Error(w, "instance failed to wake", http.StatusServiceUnavailable) |
| 346 | } |
| 347 | |
| 348 | // cleanupFailedWake stops a failed wake's process and gives its lease back, |
| 349 | // recording the outcome on w.stopOutcome. rt is the route the wake spawned |
| 350 | // against, which the termination proof needs. |
| 351 | // |
| 352 | // The lease is given back only over a generation proven terminated (spec |
| 353 | // §7.4) whenever that proof is asked for: with generations on, when the |
| 354 | // start's own error said its kill was unproven (w.genUnproven, which only |
| 355 | // GenContainment produces and which can be on with generations off), or, |
| 356 | // whatever the switch, when the wake left a process (below). Unproven, |
| 357 | // ownership and the lease are kept in unprovenWakes, the sweep retries the |
| 358 | // stop and the proof (reapUnprovenWakesLocked), and no new wake of the host |
| 359 | // starts until it holds. |
| 360 | func (a *Service) cleanupFailedWake(ctx context.Context, rt registry.Route, w *wokenInstance, lease Lease) { |
| 361 | stopCtx, cancel := context.WithTimeout(context.Background(), actStopTimeout) |
| 362 | defer cancel() |
| 363 | // A wake that left a process is proven stopped before its lease goes, |
| 364 | // whatever the switch says: two fail-closed exceptions to dormancy. |
| 365 | // One is a leader that did not stop when asked (ErrStillRunning) or is |
| 366 | // still alive (zero-downtime plan 3, task 2 fix): giving the lease back |
| 367 | // would let another box wake the host while it still writes. The other |
| 368 | // is an exec child whose leader did stop: with the flag off it ran in |
| 369 | // carlos-exec@<host> with no g<n>, and what it forked may still run |
| 370 | // there, so that cgroup is proven empty too (failedWakeStopped's |
| 371 | // uncontainedHostEmpty; Task 15 fix round 1, ruling b; Astra IMPL-3 |
| 372 | // F6). A nested or unit-backed row's own proof is unchanged. |
| 373 | if w.proc != nil { |
| 374 | err := w.proc.Stop(stopCtx) |
| 375 | w.stopOutcome.mainStop, w.stopOutcome.mainExited = err == nil, w.proc.Exited() |
| 376 | } |
| 377 | // An aborted or failed start of the host's original run unit that root |
| 378 | // may still carry out (#292, Astra P1): cancelling the client's /start |
| 379 | // leaves a start job systemd queued, and released over it, the lease |
| 380 | // would let that job launch a writer no lease covers. The unit is |
| 381 | // stopped (a stop job replaces the queued start) and proven gone on a |
| 382 | // budget of its own first. Unproven, the lease is KEPT: retained below |
| 383 | // and retried by the reaper, and at worst left to its TTL. A stall is |
| 384 | // the safe outcome; two writers are not. |
| 385 | legacy := w.proc == nil && w.legacyUncertain && !w.genUnproven |
| 386 | if legacy { |
| 387 | if err := a.legacyRunGone(rt); err != nil { |
| 388 | log.Printf("activator: %s: the aborted start's unit is not proven stopped (%v); keeping the lease until it is", rt.Host, err) |
| 389 | w.stopOutcome.mainExited = false |
| 390 | w.stopOutcome.uncertain = true |
| 391 | u := &unprovenWake{rt: rt, gen: w.gen, lease: lease, rep: w.rep, build: w.spawnIdentity, |
| 392 | layout: w.launchLayout, legacy: true} |
| 393 | // This wake's uncertainty is this retained start's, and its |
| 394 | // proof discharges it (reapUnprovenWake; seeded pre-land |
| 395 | // review, P2). |
| 396 | w.stopOutcome.owedBy = u |
| 397 | a.mu.Lock() |
| 398 | if a.unprovenWakes == nil { |
| 399 | a.unprovenWakes = make(map[string]*unprovenWake) |
| 400 | } |
| 401 | a.unprovenWakes[rt.Host] = u |
| 402 | a.mu.Unlock() |
| 403 | return |
| 404 | } |
| 405 | // #292's whole proof held: the start's unit is stopped and its |
| 406 | // cgroup empty. That was this wake's only uncertainty; it reached |
| 407 | // no sidecar, and no replica unless it holds one, which is closed |
| 408 | // below (seeded pre-land review, P2). |
| 409 | w.stopOutcome.mainStop, w.stopOutcome.mainExited = true, true |
| 410 | w.stopOutcome.uncertain = false |
| 411 | if w.sidecar == nil { |
| 412 | w.stopOutcome.sidecarStop, w.stopOutcome.sidecarExited = true, true |
| 413 | } |
| 414 | if w.rep == nil { |
| 415 | w.stopOutcome.replicaClosed = true |
| 416 | } |
| 417 | } |
| 418 | if !legacy && (a.gens || w.genUnproven || w.proc != nil) && !a.failedWakeStopped(rt, w.gen, w.proc, w.launchLayout) { |
| 419 | // Its leader did not exit (ErrStillRunning, a failed kill) or its |
| 420 | // child is not proven empty. |
| 421 | w.stopOutcome.mainExited = false |
| 422 | w.stopOutcome.uncertain = true |
| 423 | log.Printf("activator: %s: failed wake's generation %d is not proven stopped; keeping the lease until it is", rt.Host, w.gen.N) |
| 424 | a.mu.Lock() |
| 425 | if a.unprovenWakes == nil { |
| 426 | a.unprovenWakes = make(map[string]*unprovenWake) |
| 427 | } |
| 428 | // A worker's staged identity moves with it (final review F12): the |
| 429 | // failed wake's entry is about to go, and the generations file must |
| 430 | // keep naming the build its retained process may still run. |
| 431 | a.unprovenWakes[rt.Host] = &unprovenWake{rt: rt, gen: w.gen, proc: w.proc, lease: lease, rep: w.rep, build: w.spawnIdentity, |
| 432 | layout: w.launchLayout} |
| 433 | a.mu.Unlock() |
| 434 | return |
| 435 | } |
| 436 | // A backend failure may conceal resources beyond the returned handle. |
| 437 | // Successful cleanup of that handle does not erase the uncertainty. |
| 438 | if w.rep != nil { |
| 439 | closeCtx, closeCancel := context.WithTimeout(context.Background(), 30*time.Second) |
| 440 | defer closeCancel() |
| 441 | w.stopOutcome.replicaClosed = w.rep.Close(closeCtx) == nil |
| 442 | w.stopOutcome.leaseReleased = w.stopOutcome.replicaClosed |
| 443 | return |
| 444 | } |
| 445 | w.stopOutcome.leaseReleased = a.giveBack(ctx, rt.Host, lease) |
| 446 | } |
| 447 | |
amadan