| 1 | // opset.go is the entry's operation set and its barrier (zero-downtime |
| 2 | // deploys spec §7.4). The current generation stays where it always was |
| 3 | // (e.gen, e.proc). The set holds every OTHER generation of the entry that |
| 4 | // is spawning, health-checking or draining, and each is tracked until its |
| 5 | // termination is proven. Every terminal stop joins the set before its |
| 6 | // final checkpoint and lease release. |
| 7 | package activator |
| 8 | |
| 9 | import ( |
| 10 | "context" |
| 11 | "errors" |
| 12 | "fmt" |
| 13 | "log" |
| 14 | "time" |
| 15 | |
| 16 | "github.com/carlosframework/platform/internal/cgroup" |
| 17 | "github.com/carlosframework/platform/internal/instancectl" |
| 18 | "github.com/carlosframework/platform/internal/registry" |
| 19 | "github.com/carlosframework/platform/internal/release" |
| 20 | ) |
| 21 | |
| 22 | type genRole int |
| 23 | |
| 24 | const ( |
| 25 | roleSpawning genRole = iota // registered; StartGen has not returned |
| 26 | roleSuccessor // spawned; health, settle or switch pending |
| 27 | roleRetiring // was current; SIGTERM sent or about to be |
| 28 | ) |
| 29 | |
| 30 | func (r genRole) String() string { |
| 31 | switch r { |
| 32 | case roleSpawning: |
| 33 | return "spawning" |
| 34 | case roleSuccessor: |
| 35 | return "successor" |
| 36 | case roleRetiring: |
| 37 | return "retiring" |
| 38 | } |
| 39 | return "unknown" |
| 40 | } |
| 41 | |
| 42 | // genMember is one generation in an entry's operation set. Every field is |
| 43 | // written under a.mu. Its owner (the Swap that created it) stops it, and |
| 44 | // leaveSet removes it once its termination is proven. done closes at that |
| 45 | // moment, and that is what the barrier waits on. |
| 46 | type genMember struct { |
| 47 | gen Gen |
| 48 | role genRole |
| 49 | proc Proc // nil while spawning |
| 50 | cancel context.CancelFunc // cancels the attempt (spawning, successor); nil for retiring |
| 51 | done chan struct{} |
| 52 | // unproven: the owner's stop could not prove termination. The member |
| 53 | // stays in the set and reapMembersLocked finishes it. |
| 54 | unproven bool |
| 55 | reaping bool |
| 56 | retriedAt time.Time |
| 57 | // build is the worker build a shed respawn child exec'd (final review |
| 58 | // F12). The generations file lists it while the member is in the set, |
| 59 | // since a retained child may still run those bytes; leaveSet publishes |
| 60 | // again once it has left. |
| 61 | build release.StagedIdentity |
| 62 | } |
| 63 | |
| 64 | // straySidecar is a legacy sidecar whose termination is unproven, with the |
| 65 | // route its proof needs (Task 7). reaping marks a re-proof in flight. |
| 66 | type straySidecar struct { |
| 67 | proc Proc |
| 68 | rt registry.Route |
| 69 | reaping bool |
| 70 | } |
| 71 | |
| 72 | // opSet is an entry's operation set. raised records that the parent's |
| 73 | // limit may be above the tier (spec §7.3): it is set before a raise is |
| 74 | // asked, since a raise that failed partway may still have changed the |
| 75 | // parent, and cleared only by a restore that succeeded. The last member to |
| 76 | // leave restores it; a restore that fails keeps raised, and the sweep |
| 77 | // retries it (retryParentRestoreLocked, final review F11). While raised is |
| 78 | // set with no member, no overlap is admitted. restoring marks a retry in |
| 79 | // flight, and restoreDone closes when it has finished: the retry is a |
| 80 | // lifecycle operation of the entry, joined by the barrier like a member |
| 81 | // (final review R2-2). |
| 82 | type opSet struct { |
| 83 | members map[uint64]*genMember |
| 84 | raised bool |
| 85 | restoring bool |
| 86 | restoreDone chan struct{} |
| 87 | } |
| 88 | |
| 89 | // opJoinBound bounds the barrier's wait. A member's owner needs at most a |
| 90 | // cancelled spawn (bounded by the control channel's deadline) plus one |
| 91 | // bounded stop. A var so tests can shrink it. |
| 92 | var opJoinBound = instancectl.DeadlineControl + actStopTimeout |
| 93 | |
| 94 | // addMemberLocked registers m. The caller holds a.mu. |
| 95 | func (a *Service) addMemberLocked(e *actEntry, m *genMember) { |
| 96 | if e.ops.members == nil { |
| 97 | e.ops.members = make(map[uint64]*genMember) |
| 98 | } |
| 99 | e.ops.members[m.gen.N] = m |
| 100 | } |
| 101 | |
| 102 | // isMember reports whether m is still in e's set. |
| 103 | func (a *Service) isMember(e *actEntry, m *genMember) bool { |
| 104 | a.mu.Lock() |
| 105 | defer a.mu.Unlock() |
| 106 | return e != nil && e.ops.members[m.gen.N] == m |
| 107 | } |
| 108 | |
| 109 | // joinOps is the barrier (spec §7.4). It cancels every member that is still |
| 110 | // an attempt (spawning or successor) and waits, bounded by opJoinBound, for |
| 111 | // every member to leave the set, and for a parent restore retry in flight |
| 112 | // to finish (final review R2-2): its write is keyed by host, so once the |
| 113 | // entry is gone it could lower a later entry's raised parent. It reports |
| 114 | // false when a member's termination was not proven, or the retry did not |
| 115 | // finish, in time. stop() then keeps the lease, exactly as it does for |
| 116 | // ErrStillRunning. An empty set returns true at once. |
| 117 | func (a *Service) joinOps(e *actEntry) bool { |
| 118 | a.mu.Lock() |
| 119 | waits := make([]chan struct{}, 0, len(e.ops.members)+1) |
| 120 | for _, m := range e.ops.members { |
| 121 | if m.cancel != nil { |
| 122 | m.cancel() |
| 123 | } |
| 124 | waits = append(waits, m.done) |
| 125 | } |
| 126 | if e.ops.restoreDone != nil { |
| 127 | waits = append(waits, e.ops.restoreDone) |
| 128 | } |
| 129 | hook := a.joinWaiting |
| 130 | a.mu.Unlock() |
| 131 | if len(waits) == 0 { |
| 132 | return true |
| 133 | } |
| 134 | if hook != nil { |
| 135 | hook(e.host, "operation set") |
| 136 | } |
| 137 | bound := time.NewTimer(opJoinBound) |
| 138 | defer bound.Stop() |
| 139 | for _, w := range waits { |
| 140 | select { |
| 141 | case <-w: |
| 142 | case <-bound.C: |
| 143 | return false |
| 144 | } |
| 145 | } |
| 146 | return true |
| 147 | } |
| 148 | |
| 149 | // genKiller is plan 2's generation kill (ExecBackend.KillGen and its |
| 150 | // SplitBackend forward): it kills what is left of generation n's own child |
| 151 | // cgroup and returns once that child is empty. Plan 3 owns the name; plan 5 |
| 152 | // uses it and does not declare its own. |
| 153 | type genKiller interface { |
| 154 | KillGen(ctx context.Context, rt registry.Route, n uint64) error |
| 155 | } |
| 156 | |
| 157 | // hostKiller is the exec backend's host-cgroup kill: cgroup.kill on host's |
| 158 | // own cgroup, then wait until it is empty. It proves a worker generation |
| 159 | // terminated (plan 2 D4 keeps workers there). |
| 160 | type hostKiller interface { |
| 161 | KillHost(ctx context.Context, rt registry.Route) error |
| 162 | } |
| 163 | |
| 164 | // hostEmptier is the exec backend's host-cgroup proof for a flag-off |
| 165 | // terminal stop (Task 8, F6): host's own cgroup is read first and killed |
| 166 | // only when something is left, then waited empty. stopHostProven prefers |
| 167 | // it to hostKiller, so a clean stop leaves no kill mark. |
| 168 | type hostEmptier interface { |
| 169 | EmptyHost(ctx context.Context, rt registry.Route) error |
| 170 | } |
| 171 | |
| 172 | // stopMember stops m's process, bounded by actStopTimeout: SIGTERM, the |
| 173 | // generation's own graceful drain, then SIGKILL (spec §7.5 step 6). It |
| 174 | // reports whether termination is proven, which takes two things: the |
| 175 | // leader has exited, and the generation's child cgroup is empty (spec |
| 176 | // §7.3, §7.4). The flat childProc signals only its leader, so a process |
| 177 | // the app forked can outlive it in g<n>. |
| 178 | // |
| 179 | // A member with no process never got one back from its spawn. That is not |
| 180 | // proof: a nested start whose kill was unproven answers no process |
| 181 | // (startUnitAt), so its generation's child is proven empty here instead. |
| 182 | // Only a member that never had a number is trivially proven. |
| 183 | func (a *Service) stopMember(e *actEntry, m *genMember) bool { |
| 184 | a.mu.Lock() |
| 185 | rt := e.route |
| 186 | a.mu.Unlock() |
| 187 | if m.proc == nil { |
| 188 | return a.handlelessGenEmpty(rt, m.gen) |
| 189 | } |
| 190 | var err error |
| 191 | if cp, ok := m.proc.(*claimProcess); ok && m.role == roleRetiring { |
| 192 | // A serving claims-on generation drains like a flat one (spec §7.5 |
| 193 | // step 6, index P6 #4): SIGTERM and actStopTimeout first, then |
| 194 | // KillAndEmpty and the proof, with their own bound. An unready |
| 195 | // candidate takes the arm below: Stop kills it at once. |
| 196 | ctx, cancel := context.WithTimeout(context.Background(), actStopTimeout+instancectl.DeadlineControl) |
| 197 | err = cp.Retire(ctx, actStopTimeout) |
| 198 | cancel() |
| 199 | } else { |
| 200 | ctx, cancel := context.WithTimeout(context.Background(), actStopTimeout) |
| 201 | err = m.proc.Stop(ctx) |
| 202 | cancel() |
| 203 | } |
| 204 | if errors.Is(err, ErrStillRunning) { |
| 205 | return false |
| 206 | } |
| 207 | if err != nil { |
| 208 | log.Printf("activator: stop generation %d: %v", m.gen.N, err) |
| 209 | } |
| 210 | return a.genTerminated(rt, m.gen, m.proc) // swapsim:M7 |
| 211 | } |
| 212 | |
| 213 | // handlelessGenEmpty is the termination proof for a generation with no |
| 214 | // process handle (a member that never spawned, a failed wake whose start |
| 215 | // returned none): its own child, proven empty through KillGen. A number of |
| 216 | // 0 names no child, so it is trivially proven for a member (nothing was |
| 217 | // ever asked to start under it). Once proven, the generation's own socket |
| 218 | // directory goes if it is empty (removeGenDir, rev 19 rule 3). |
| 219 | func (a *Service) handlelessGenEmpty(rt registry.Route, g Gen) bool { |
| 220 | if g.N == 0 { |
| 221 | return true |
| 222 | } |
| 223 | if !a.genChildEmpty(rt, &genMember{gen: g}) { |
| 224 | return false |
| 225 | } |
| 226 | removeGenDir(rt, g) |
| 227 | return true |
| 228 | } |
| 229 | |
| 230 | // genTerminated is the ONE termination proof for a generation (spec §7.3, |
| 231 | // §7.4): its leader has exited AND its child is proven empty. Every path |
| 232 | // that would give up a generation's lease or its place in the operation |
| 233 | // set asks it: a member's stop, the reaper, the current generation's stop, |
| 234 | // a pending stop's finish, a failed wake and its retained retry. A nil |
| 235 | // process never ran. |
| 236 | // |
| 237 | // Once it holds for a process that ran, the generation's own socket |
| 238 | // directory goes if it is empty (removeGenDir, rev 19 rule 3). Its socket |
| 239 | // is gone by then unless the app left it behind, and the agent's control |
| 240 | // listener closed with the stop. A nil process proves nothing about a |
| 241 | // directory, so it is left; a leftover is harmless (rule 3). |
| 242 | func (a *Service) genTerminated(rt registry.Route, g Gen, p Proc) bool { |
| 243 | if p == nil { |
| 244 | if g.Worker { |
| 245 | // A worker's host cgroup is proven whether or not a process |
| 246 | // handle is left: a dead leader's descendants live there (Task 11). |
| 247 | return a.genChildEmpty(rt, &genMember{gen: g}) |
| 248 | } |
| 249 | return true |
| 250 | } |
| 251 | if !p.Exited() { |
| 252 | return false |
| 253 | } |
| 254 | if !a.genChildEmpty(rt, &genMember{gen: g, proc: p}) { |
| 255 | return false |
| 256 | } |
| 257 | removeGenDir(rt, g) |
| 258 | return true |
| 259 | } |
| 260 | |
| 261 | // genChildEmpty proves m's generation child empty, after its leader has |
| 262 | // exited, through plan 2's KillGen: whatever is left in g<n> (or the |
| 263 | // nested carlos-instance-gen@<id>:<n> unit) is killed and the child is |
| 264 | // waited empty. That kill is the only one that can reach a descendant the |
| 265 | // leader left behind, and it touches no other generation. |
| 266 | // |
| 267 | // A backend with no generation kill (a test double), or one that answers |
| 268 | // errNoGenContainment, has no generation children: the generation ran in |
| 269 | // the host cgroup, and the terminal stop's own stop evidence on that cgroup |
| 270 | // is the proof (plan 2, "stop evidence on the parent"). Any other error is |
| 271 | // an unproven termination. A retained failed wake has no such terminal stop |
| 272 | // behind it, so failedWakeStopped asks for the host cgroup itself |
| 273 | // (genChildProof's uncontained). |
| 274 | func (a *Service) genChildEmpty(rt registry.Route, m *genMember) bool { |
| 275 | proven, _ := a.genChildProof(rt, m) |
| 276 | return proven |
| 277 | } |
| 278 | |
| 279 | // genChildProof is genChildEmpty's proof. uncontained reports that it |
| 280 | // proved nothing beyond the leader's exit, because the generation has no |
| 281 | // child of its own: the backend has no generation kill, KillGen answered |
| 282 | // errNoGenContainment (the flag-off ExecBackend), or it has no number (a |
| 283 | // flag-off worker). Such a generation ran in carlos-exec@<host>, where a |
| 284 | // process its leader forked can outlive it (Task 15, Astra IMPL-3 F6). |
| 285 | func (a *Service) genChildProof(rt registry.Route, m *genMember) (proven, uncontained bool) { |
| 286 | if cp, ok := m.proc.(*claimProcess); ok { |
| 287 | // A claims-on generation (Task 10): its own Stop is the proof, and |
| 288 | // only a Stop that completed (KillAndEmpty, the exit, the settle and |
| 289 | // the close) counts. A leader that exited after KillAndEmpty failed |
| 290 | // is not terminated. |
| 291 | ctx, cancel := context.WithTimeout(context.Background(), instancectl.DeadlineControl) |
| 292 | defer cancel() |
| 293 | return cp.provenStopped(ctx), false |
| 294 | } |
| 295 | if m.gen.Worker { |
| 296 | // A worker runs in its host cgroup (plan 2 D4): that cgroup, not a |
| 297 | // g<n> it never ran in, is what must be empty (spec §7.4, §7.9). |
| 298 | hk, ok := a.backend.(hostKiller) |
| 299 | if !ok { |
| 300 | return true, false // a backend with no host cgroup: the leader's exit is all there is |
| 301 | } |
| 302 | ctx, cancel := context.WithTimeout(context.Background(), instancectl.DeadlineControl) |
| 303 | defer cancel() |
| 304 | if err := hk.KillHost(ctx, rt); err != nil { |
| 305 | log.Printf("activator: %s: worker generation %d's host cgroup is not proven empty: %v", rt.Host, m.gen.N, err) |
| 306 | return false, false |
| 307 | } |
| 308 | return true, false |
| 309 | } |
| 310 | k, ok := a.backend.(genKiller) |
| 311 | if !ok || m.gen.N == 0 { |
| 312 | return true, true |
| 313 | } |
| 314 | ctx, cancel := context.WithTimeout(context.Background(), instancectl.DeadlineControl) |
| 315 | defer cancel() |
| 316 | err := k.KillGen(ctx, rt, m.gen.N) |
| 317 | if err == nil { |
| 318 | return true, false |
| 319 | } |
| 320 | if errors.Is(err, errNoGenContainment) { |
| 321 | return true, true |
| 322 | } |
| 323 | log.Printf("activator: %s: generation %d's child is not proven empty: %v", rt.Host, m.gen.N, err) |
| 324 | return false, false |
| 325 | } |
| 326 | |
| 327 | // uncontainedHostEmpty proves a failed wake's host cgroup empty when its |
| 328 | // generation had no child of its own (Task 15, Astra IMPL-3 F6, flag off): |
| 329 | // KillHost kills whatever is left in carlos-exec@<host> and waits it |
| 330 | // empty, as the worker proof does. An absent cgroup is empty. Only an exec |
| 331 | // child's wake ran there (hostCgroupIsTheWakes); a nested or unit-backed |
| 332 | // row's proof is its own, and a cgroup that is not the row's is never |
| 333 | // killed (fix round 1, ruling c). |
| 334 | // |
| 335 | // Killing is safe here because nothing of another entry can be in that |
| 336 | // cgroup at this point. The failed wake still holds the host's lease, and |
| 337 | // while it is retained the host's next wake, request or claim, is refused |
| 338 | // (errWakeUnproven); the failed wake never reached the legacy sidecar's |
| 339 | // co-spawn, which comes after every failure that reaches this cleanup; |
| 340 | // sidecar and seed starts run only under the host's live entry; workers |
| 341 | // and sidecar members are rows of their own, each with its own |
| 342 | // carlos-exec@<host>. What can be there is the wake's own leftovers, or a |
| 343 | // legacy sidecar of an earlier entry whose stop did not end it. With the |
| 344 | // flag off such a sidecar is untracked (straySidecars is filled only with |
| 345 | // generations on), and it is meant to be gone, so killing it is right. The |
| 346 | // killed cgroup is recreated by the next start into it (Task 19). |
| 347 | // |
| 348 | // A box with no delegated cgroup (cgroup.ErrNoDelegation) ran the wake |
| 349 | // unaccounted, with nothing that could reach a descendant: the leader's |
| 350 | // exit is all there is, as before. Keeping the lease there would refuse |
| 351 | // the host on this box until the agent restarts while proving nothing. |
| 352 | // Any other failure keeps the wake retained, and the sweep asks again. |
| 353 | func (a *Service) uncontainedHostEmpty(rt registry.Route, g Gen, launched Layout) bool { |
| 354 | if !a.hostCgroupIsTheWakes(rt, g, launched) { |
| 355 | return true // nested or unit-backed: its own proof already ran |
| 356 | } |
| 357 | hk, ok := a.backend.(hostKiller) |
| 358 | if !ok { |
| 359 | return true // a backend with no host cgroup: the leader's exit is all there is |
| 360 | } |
| 361 | ctx, cancel := context.WithTimeout(context.Background(), instancectl.DeadlineControl) |
| 362 | defer cancel() |
| 363 | err := hk.KillHost(ctx, rt) |
| 364 | if err == nil { |
| 365 | return true |
| 366 | } |
| 367 | if errors.Is(err, cgroup.ErrNoDelegation) { |
| 368 | log.Printf("activator: %s: failed wake's generation %d ran with no delegated cgroup; its leader's exit is the only proof there is", rt.Host, g.N) |
| 369 | return true |
| 370 | } |
| 371 | log.Printf("activator: %s: failed wake's generation %d had no cgroup of its own, and its host cgroup is not proven empty: %v", rt.Host, g.N, err) |
| 372 | return false |
| 373 | } |
| 374 | |
| 375 | // stopHostProven is the flag-off half of F6 for a terminal stop (Task 8): |
| 376 | // a flat or foreign exec row's carlos-exec@<host> is proven empty before |
| 377 | // the stop releases the lease. The leader's exit says nothing of a process |
| 378 | // it forked, and with the flag off that process sits in the host cgroup |
| 379 | // with no generation child to reach it. hostEmptier when the backend has |
| 380 | // it (read first, kill only when populated), else hostKiller; a backend |
| 381 | // with neither has no host cgroup, and the leader's exit is all there is. |
| 382 | // A claims-on process proved its own stop; a nested or unit-backed row has |
| 383 | // its own proof (hostCgroupIsTheWakes, on the layout the wake launched). |
| 384 | // |
| 385 | // Killing is safe here for uncontainedHostEmpty's reasons: the entry is |
| 386 | // stopping, its sidecar was stopped first, a supervised respawn and a |
| 387 | // sidecar start are joined before this runs, and nothing of another entry |
| 388 | // lives in that cgroup. A box with no delegated cgroup |
| 389 | // (cgroup.ErrNoDelegation) keeps the leader-only proof with a log line |
| 390 | // (ledger 127). Any other failure answers false, and the stop stays |
| 391 | // pending with its lease. Never called with a.mu held. |
| 392 | func (a *Service) stopHostProven(e *actEntry) bool { |
| 393 | a.mu.Lock() |
| 394 | rt, g, p, launched := e.route, e.gen, e.proc, e.launchLayout |
| 395 | a.mu.Unlock() |
| 396 | if _, ok := p.(*claimProcess); ok { |
| 397 | return true |
| 398 | } |
| 399 | if !a.hostCgroupIsTheWakes(rt, g, launched) { |
| 400 | return true |
| 401 | } |
| 402 | ctx, cancel := context.WithTimeout(context.Background(), instancectl.DeadlineControl) |
| 403 | defer cancel() |
| 404 | var err error |
| 405 | if he, ok := a.backend.(hostEmptier); ok { |
| 406 | err = he.EmptyHost(ctx, rt) |
| 407 | } else if hk, ok := a.backend.(hostKiller); ok { |
| 408 | err = hk.KillHost(ctx, rt) |
| 409 | } else { |
| 410 | return true // a backend with no host cgroup: the leader's exit is all there is |
| 411 | } |
| 412 | if err == nil { |
| 413 | return true |
| 414 | } |
| 415 | if errors.Is(err, cgroup.ErrNoDelegation) { |
| 416 | log.Printf("activator: %s: stopped with no delegated cgroup; its leader's exit is the only proof there is", rt.Host) |
| 417 | return true |
| 418 | } |
| 419 | log.Printf("activator: stop %s: its host cgroup is not proven empty: %v", rt.Host, err) |
| 420 | return false |
| 421 | } |
| 422 | |
| 423 | // wakeHostProven is a flag-off wake's proof, under its lease and before |
| 424 | // Restore (Task 8 fix round 1, finding 4), that carlos-exec@<host> holds |
| 425 | // nothing: EmptyHost reads cgroup.events and kills only what is left, so a |
| 426 | // clean host costs one read and leaves no kill mark. Only a row whose child |
| 427 | // runs there is asked: an exec child (flat or foreign) or a worker, which |
| 428 | // spawns there whatever its layout says; a nested or unit-backed row runs |
| 429 | // elsewhere and has its own proofs. A backend with no hostEmptier has no |
| 430 | // host cgroup to read. No delegated cgroup (cgroup.ErrNoDelegation) is |
| 431 | // today's wake. Any other failure is errHostUnproven. |
| 432 | // |
| 433 | // Killing is safe here for uncontainedHostEmpty's reasons: the wake holds |
| 434 | // the host's lease, nothing of it has run yet, and the stop that released |
| 435 | // the lease before it proved its own processes gone, so whatever is left |
| 436 | // belongs to no live entry. Never called with a.mu held. |
| 437 | func (a *Service) wakeHostProven(ctx context.Context, e *actEntry, rt registry.Route) error { |
| 438 | he, ok := a.backend.(hostEmptier) |
| 439 | if !ok || rt.Backing == registry.BackingUnit { |
| 440 | return nil |
| 441 | } |
| 442 | if !e.worker { |
| 443 | if c, ok := a.backend.(Classifier); ok { |
| 444 | if l, _ := c.Layout(rt); l == LayoutNested { |
| 445 | return nil |
| 446 | } |
| 447 | } |
| 448 | } |
| 449 | pctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), instancectl.DeadlineControl) |
| 450 | defer cancel() |
| 451 | err := he.EmptyHost(pctx, rt) |
| 452 | if err == nil || errors.Is(err, cgroup.ErrNoDelegation) { |
| 453 | return nil |
| 454 | } |
| 455 | return fmt.Errorf("%w: %v", errHostUnproven, err) |
| 456 | } |
| 457 | |
| 458 | // hostCgroupIsTheWakes reports whether a failed wake of rt may have run in |
| 459 | // carlos-exec@<host>, so that cgroup must be proven empty: an exec child, |
| 460 | // that is, a flat or foreign row. A unit-backed row runs under systemd |
| 461 | // (SplitBackend's unit side), and a nested row runs as a carlos-instance |
| 462 | // unit in its host slice; both have their own proof. A worker spawns into |
| 463 | // carlos-exec@<host> whatever its layout says (StartWorker), and a |
| 464 | // flag-off worker's wake is the only one with no number (an instance wake |
| 465 | // is always numbered, nextGen), so a number of 0 counts as an exec child. |
| 466 | // rt is the row the wake launched against, so its backing is the launch's. |
| 467 | // |
| 468 | // The layout is the one the launch used (launched), not the one the row |
| 469 | // reads now (final review round 1, finding 6, Astra): a flat wake whose |
| 470 | // layout reads fenced or conflicting by cleanup time still ran in |
| 471 | // carlos-exec@<host>. Only a launch that was nested skips the proof, and |
| 472 | // only while the row still reads nested: a launch layout that is unknown |
| 473 | // (a backend that cannot say, "" here), fenced or conflicting, or a |
| 474 | // recorded and a derived layout that disagree, requires it. |
| 475 | func (a *Service) hostCgroupIsTheWakes(rt registry.Route, g Gen, launched Layout) bool { |
| 476 | if rt.Backing == registry.BackingUnit { |
| 477 | return false |
| 478 | } |
| 479 | if g.N == 0 || g.Worker { |
| 480 | return true |
| 481 | } |
| 482 | if launched != LayoutNested { |
| 483 | return true |
| 484 | } |
| 485 | c, ok := a.backend.(Classifier) |
| 486 | if !ok { |
| 487 | return true |
| 488 | } |
| 489 | l, _ := c.Layout(rt) |
| 490 | return l != LayoutNested |
| 491 | } |
| 492 | |
| 493 | // dropMember stops m and takes it out of e's set once its termination is |
| 494 | // proven. It reports whether termination was proven. An unproven member |
| 495 | // stays in the set, marked, and the sweep's reaper finishes it. |
| 496 | func (a *Service) dropMember(e *actEntry, m *genMember) bool { |
| 497 | proven := a.stopMember(e, m) // swapsim:M4 |
| 498 | if !proven { |
| 499 | a.mu.Lock() |
| 500 | m.unproven, m.cancel, m.retriedAt = true, nil, a.now() |
| 501 | a.mu.Unlock() |
| 502 | log.Printf("activator: generation %d of %s did not prove its exit; keeping it in the operation set", m.gen.N, e.host) |
| 503 | return false |
| 504 | } |
| 505 | a.leaveSet(e, m) |
| 506 | return true |
| 507 | } |
| 508 | |
| 509 | // leaveSet removes a member whose termination is proven. When it is the |
| 510 | // last member and the parent was raised, the parent is restored FIRST, |
| 511 | // while the member still holds the set. That way no new attempt can raise |
| 512 | // the parent again in between (spec §7.3: the restore happens after the |
| 513 | // retiring child is empty). The restore is to the CURRENT generation's |
| 514 | // captured tier (spec §7.3): the incumbent's after a failed successor, the |
| 515 | // successor's after a switch. done closes last. |
| 516 | // |
| 517 | // A restore that fails keeps raised (final review F11): the member still |
| 518 | // leaves, since its termination is proven, but the obligation stays on the |
| 519 | // entry and the sweep retries it (retryParentRestoreLocked). |
| 520 | func (a *Service) leaveSet(e *actEntry, m *genMember) { |
| 521 | a.mu.Lock() |
| 522 | restore := e.ops.raised && len(e.ops.members) == 1 && e.ops.members[m.gen.N] == m |
| 523 | rt, tier := e.route, e.gen.Tier |
| 524 | a.mu.Unlock() |
| 525 | restored := false |
| 526 | if restore { |
| 527 | restored = a.restoreParent(rt, tier) |
| 528 | } |
| 529 | a.mu.Lock() |
| 530 | if restored { |
| 531 | e.ops.raised = false |
| 532 | } |
| 533 | if e.ops.members[m.gen.N] == m { |
| 534 | delete(e.ops.members, m.gen.N) |
| 535 | } |
| 536 | a.mu.Unlock() |
| 537 | close(m.done) |
| 538 | if _, listed := a.genIdentity(m.gen); listed || m.build != (release.StagedIdentity{}) { |
| 539 | // Its child is proven gone: its build is no longer referenced. |
| 540 | _ = a.persistGenerations() |
| 541 | } |
| 542 | } |
| 543 | |
| 544 | // restoreParent puts rt's parent back at tier, and reports whether it did. |
| 545 | // A backend without the swap seam has no parent to restore. Never call it |
| 546 | // with a.mu held. |
| 547 | func (a *Service) restoreParent(rt registry.Route, tier cgroup.Limit) bool { |
| 548 | gb, ok := a.backend.(SwapBackend) |
| 549 | if !ok { |
| 550 | return true |
| 551 | } |
| 552 | rctx, cancel := context.WithTimeout(context.Background(), instancectl.DeadlineControl) |
| 553 | defer cancel() |
| 554 | if err := gb.RestoreGenParent(rctx, rt, tier); err != nil && !parentUntouched(err) { |
| 555 | log.Printf("activator: restore parent limit for %s: %v (kept; the sweep retries)", rt.Host, err) |
| 556 | return false |
| 557 | } |
| 558 | return true |
| 559 | } |
| 560 | |
| 561 | // parentUntouched reports whether a parent-limit error says no parent |
| 562 | // limit is in this box's hands at all: no generation containment |
| 563 | // (errNoGenContainment) or no delegated cgroup manager. Both are refused |
| 564 | // before any write, and both are fixed for the agent's life, so a raise |
| 565 | // they refuse changed nothing and a restore they refuse has nothing to |
| 566 | // put back. |
| 567 | func parentUntouched(err error) bool { |
| 568 | return errors.Is(err, errNoGenContainment) || errors.Is(err, cgroup.ErrNoDelegation) |
| 569 | } |
| 570 | |
| 571 | // retryParentRestoreLocked retries a parent restore that failed when the |
| 572 | // last member left (final review F11). It runs only while the set is empty |
| 573 | // and no retry is in flight; the restore itself runs off a.mu. An entry |
| 574 | // that is gone takes its obligation with it: the next cold spawn under an |
| 575 | // empty parent writes the tier afresh (cgroup.ensureParent). The caller |
| 576 | // holds a.mu. |
| 577 | // |
| 578 | // The retry is a lifecycle operation of the entry (final review R2-2). Only |
| 579 | // an awake entry that is still the host's launches one, so a stopping entry |
| 580 | // launches none; and a terminal stop, a hold's included, joins one in |
| 581 | // flight (joinOps, through restoreDone) before it releases the lease, |
| 582 | // forgets the entry or installs a pinned replacement. The launch and the |
| 583 | // state change to stopping are both under a.mu, so a stop either sees the |
| 584 | // retry or the retry is never launched. |
| 585 | func (a *Service) retryParentRestoreLocked(e *actEntry) { |
| 586 | if !e.ops.raised || e.ops.restoring || len(e.ops.members) > 0 { |
| 587 | return |
| 588 | } |
| 589 | if a.entries[e.host] != e || e.state != actAwake { |
| 590 | return |
| 591 | } |
| 592 | done := make(chan struct{}) |
| 593 | e.ops.restoring, e.ops.restoreDone = true, done |
| 594 | rt, tier := e.route, e.gen.Tier |
| 595 | go func() { |
| 596 | ok := a.restoreParent(rt, tier) |
| 597 | a.mu.Lock() |
| 598 | e.ops.restoring, e.ops.restoreDone = false, nil |
| 599 | if ok && len(e.ops.members) == 0 { |
| 600 | e.ops.raised = false |
| 601 | } |
| 602 | a.mu.Unlock() |
| 603 | close(done) |
| 604 | }() |
| 605 | } |
| 606 | |
| 607 | // reapMembersLocked finishes members whose owner could not prove their stop. |
| 608 | // Once a member's process has exited (or it never had one), the reaper |
| 609 | // proves its generation child empty (reapExited) and only then takes it out |
| 610 | // of the set. Until the leader exits it re-issues Stop every |
| 611 | // actStopTimeout, the cadence finishPendingStops keeps for the main |
| 612 | // process. The caller holds a.mu. |
| 613 | func (a *Service) reapMembersLocked(e *actEntry) { |
| 614 | for _, m := range e.ops.members { |
| 615 | if !m.unproven || m.reaping { |
| 616 | continue |
| 617 | } |
| 618 | if m.proc == nil || m.proc.Exited() { |
| 619 | m.reaping = true |
| 620 | go a.reapExited(e, m) |
| 621 | continue |
| 622 | } |
| 623 | if a.now().Sub(m.retriedAt) < actStopTimeout { |
| 624 | continue |
| 625 | } |
| 626 | m.reaping, m.retriedAt = true, a.now() |
| 627 | go func(m *genMember) { |
| 628 | ctx, cancel := context.WithTimeout(context.Background(), actStopTimeout) |
| 629 | err := m.proc.Stop(ctx) |
| 630 | cancel() |
| 631 | if err != nil && !errors.Is(err, ErrStillRunning) { |
| 632 | log.Printf("activator: retry stop of generation %d of %s: %v", m.gen.N, e.host, err) |
| 633 | } |
| 634 | a.mu.Lock() |
| 635 | m.reaping = false |
| 636 | a.mu.Unlock() |
| 637 | }(m) |
| 638 | } |
| 639 | } |
| 640 | |
| 641 | // reapExited finishes an unproven member whose leader has exited, or that |
| 642 | // never had one: it leaves the set only once its generation child is |
| 643 | // proven empty. A failed proof leaves it marked, and the next sweep tries |
| 644 | // the kill again. |
| 645 | func (a *Service) reapExited(e *actEntry, m *genMember) { |
| 646 | a.mu.Lock() |
| 647 | rt := e.route |
| 648 | a.mu.Unlock() |
| 649 | var proven bool |
| 650 | if m.proc == nil { |
| 651 | proven = a.handlelessGenEmpty(rt, m.gen) |
| 652 | } else { |
| 653 | proven = a.genTerminated(rt, m.gen, m.proc) |
| 654 | } |
| 655 | if proven { |
| 656 | a.leaveSet(e, m) |
| 657 | return |
| 658 | } |
| 659 | a.mu.Lock() |
| 660 | m.reaping, m.retriedAt = false, a.now() |
| 661 | a.mu.Unlock() |
| 662 | } |
| 663 | |
| 664 | // The CURRENT generation is held to the same proof (spec §7.4: "each is |
| 665 | // tracked until its termination is proven"). stop() and a pending stop's |
| 666 | // finish prove its child empty before the final checkpoint and the lease |
| 667 | // release, and a failed wake proves its generation's child empty before it |
| 668 | // gives the lease back. |
| 669 | |
| 670 | // releasable is THE retain-until-proven rule for an entry's lease (spec |
| 671 | // §7.4): a stop finishes, and its lease is released, only when every |
| 672 | // member has left the operation set, no parent restore retry is in flight |
| 673 | // (final review R2-2), no stray sidecar of the host is tracked (Task 7), |
| 674 | // the current generation has passed genTerminated (its leader exited and |
| 675 | // its child, or a worker's host cgroup, empty), and every run of the |
| 676 | // instance is proven stopped, its unit stopped and its cgroup empty |
| 677 | // (settleCmds; run spec §5.3, merge ruling M1). A run writes the database |
| 678 | // as the serving process does, so it is held to the same rule. It runs |
| 679 | // without a.mu held; genTerminated may take DeadlineControl, and a run's |
| 680 | // settle up to cmdStopBound. |
| 681 | func (a *Service) releasable(e *actEntry) bool { |
| 682 | a.mu.Lock() |
| 683 | members, strays, restoring := len(e.ops.members), len(a.straySidecars[e.host]), e.ops.restoring |
| 684 | rt, g, p := e.route, e.gen, e.proc |
| 685 | a.mu.Unlock() |
| 686 | if members > 0 || strays > 0 || restoring { |
| 687 | return false |
| 688 | } |
| 689 | return a.genTerminated(rt, g, p) && a.settleCmds(e) |
| 690 | } |
| 691 | |
| 692 | // keepPendingLocked leaves a stop pending with its lease held: the entry |
| 693 | // stays in stopping, and finishPendingStops finishes it once termination |
| 694 | // is proven. stopRetriedAt is this attempt, so the sweep's re-stop cadence |
| 695 | // runs a full actStopTimeout after it. The caller holds a.mu. |
| 696 | func (a *Service) keepPendingLocked(e *actEntry, outcome *stopObservation) { |
| 697 | e.pendingStop = true |
| 698 | e.stopOutcome = outcome |
| 699 | e.stopRetriedAt = a.now() |
| 700 | } |
| 701 | |
| 702 | // reapCurrentGen finishes a pending stop whose main process has exited |
| 703 | // (finishPendingStops), once its termination proof holds: with generations |
| 704 | // on, the retain-until-proven rule (releasable); with them off, every |
| 705 | // sidecar the stop kept unproven re-stopped and exited (sidecarsExited, fix |
| 706 | // wave 1, item 1), then the host cgroup proven empty (stopHostProven, Task |
| 707 | // 8, F6). A failed proof leaves the stop pending, and the next sweep tries |
| 708 | // again. The caller set e.genReaping; this clears it. Never called with |
| 709 | // a.mu held. |
| 710 | func (a *Service) reapCurrentGen(e *actEntry) { |
| 711 | proven := func(e *actEntry) bool { return a.sidecarsExited(e, true) && a.stopHostProven(e) } |
| 712 | if a.gens { |
| 713 | proven = a.releasable |
| 714 | } |
| 715 | if !proven(e) { |
| 716 | a.mu.Lock() |
| 717 | e.genReaping = false |
| 718 | a.mu.Unlock() |
| 719 | return |
| 720 | } |
| 721 | a.mu.Lock() |
| 722 | e.genReaping, e.pendingStop = false, false |
| 723 | outcome, p := e.stopOutcome, e.proc |
| 724 | a.mu.Unlock() |
| 725 | if outcome == nil { |
| 726 | outcome = &stopObservation{} |
| 727 | } |
| 728 | if p == nil { |
| 729 | // Only the barrier (flag on) or the host proof (flag off) left it |
| 730 | // pending. |
| 731 | a.finishStop(e, outcome) |
| 732 | return |
| 733 | } |
| 734 | a.finishPendingStop(e, outcome) |
| 735 | } |
| 736 | |
| 737 | // unprovenWake is a failed wake whose generation is not proven stopped |
| 738 | // (spec §7.4). Its lease (and replica, if it got one) is kept until the |
| 739 | // sweep proves the generation stopped; until then no new wake of the host |
| 740 | // starts. busy marks a retry in flight. |
| 741 | type unprovenWake struct { |
| 742 | rt registry.Route |
| 743 | gen Gen |
| 744 | proc Proc |
| 745 | lease Lease |
| 746 | rep Rep |
| 747 | busy bool |
| 748 | // build is a failed worker wake's staged identity (final review F12): |
| 749 | // the generations file keeps listing it until the proof holds, since |
| 750 | // the retained process may still run those bytes. |
| 751 | build release.StagedIdentity |
| 752 | // layout is the layout the failed wake's launch used |
| 753 | // (wokenInstance.launchLayout), which every retry of the proof decides |
| 754 | // by (finding 6). |
| 755 | layout Layout |
| 756 | // legacy: an aborted start of the host's original run unit whose |
| 757 | // queued job may still run (#292, Astra P1). Proven by legacyRunGone. |
| 758 | legacy bool |
| 759 | } |
| 760 | |
| 761 | // errWakeUnproven refuses a wake while the host's last failed wake is not |
| 762 | // proven stopped. |
| 763 | var errWakeUnproven = errors.New("the last failed wake's generation is not proven stopped") |
| 764 | |
| 765 | // errHostUnproven refuses a flag-off wake whose host cgroup holds processes |
| 766 | // it could not prove gone (wakeHostProven). |
| 767 | var errHostUnproven = errors.New("the host cgroup holds processes not proven gone") |
| 768 | |
| 769 | // genStartUnproven reports whether a generation start's error says the |
| 770 | // start failed and its kill did not prove the generation gone: plan 2's |
| 771 | // errGenUnproven (startUnitAt) or root's instancectl.ErrGenUnproven. The |
| 772 | // generation may still be running, so whatever the wake holds for the host |
| 773 | // is kept. Both exist only under GenContainment, which can be on while |
| 774 | // a.gens is off, so callers never gate this on a.gens. |
| 775 | func genStartUnproven(err error) bool { |
| 776 | return errors.Is(err, errGenUnproven) || errors.Is(err, instancectl.ErrGenUnproven) |
| 777 | } |
| 778 | |
| 779 | // failedWakeStopped is the termination proof for a failed wake's |
| 780 | // generation. With a process it is genTerminated. Without one (the start |
| 781 | // answered none, as startUnitAt does when its kill was unproven) it is the |
| 782 | // generation's own child, proven empty through KillGen, and never the |
| 783 | // missing handle: a retained wake with no number cannot be proven at all |
| 784 | // and stays retained. |
| 785 | // |
| 786 | // Either way, a generation with no child of its own (flag off) is proven |
| 787 | // only once its host cgroup is empty too (uncontainedHostEmpty; Task 15, |
| 788 | // Astra IMPL-3 F6). Its leader's exit, all genTerminated can see there, |
| 789 | // leaves whatever the leader forked running in carlos-exec@<host>. launched |
| 790 | // is the layout the wake's launch used (wokenInstance.launchLayout). |
| 791 | // |
| 792 | // Once the whole proof holds, and never before, the generation's own empty |
| 793 | // socket directory goes (removeGenDir, rev 19 rule 3). |
| 794 | func (a *Service) failedWakeStopped(rt registry.Route, g Gen, p Proc, launched Layout) bool { |
| 795 | if p == nil && g.N == 0 { |
| 796 | log.Printf("activator: %s: failed wake left no process handle and no generation number; it cannot be proven stopped, so its lease stays held", rt.Host) |
| 797 | return false |
| 798 | } |
| 799 | // genTerminated's proof (handlelessGenEmpty's without a process), |
| 800 | // spelled out so the proof can say whether it reached a child. |
| 801 | if p != nil && !p.Exited() { |
| 802 | return false |
| 803 | } |
| 804 | proven, uncontained := a.genChildProof(rt, &genMember{gen: g, proc: p}) |
| 805 | if !proven { |
| 806 | return false |
| 807 | } |
| 808 | if uncontained && !a.uncontainedHostEmpty(rt, g, launched) { |
| 809 | return false |
| 810 | } |
| 811 | // The whole proof holds, the host cgroup's included where it is needed: |
| 812 | // the generation's own socket directory goes if it is empty (rev 19 |
| 813 | // rule 3), as genTerminated does for every other path (Astra, review of |
| 814 | // 19c07b2b: a failed wake bypassed it, and directories piled up). |
| 815 | removeGenDir(rt, g) |
| 816 | return true |
| 817 | } |
| 818 | |
| 819 | // reapUnprovenWakesLocked retries each unproven failed wake's proof and, |
| 820 | // once it holds, gives the lease back. The caller holds a.mu. |
| 821 | func (a *Service) reapUnprovenWakesLocked() { |
| 822 | for host, u := range a.unprovenWakes { |
| 823 | if u.busy { |
| 824 | continue |
| 825 | } |
| 826 | // A legacy start's failed wake records its outcome as its entry |
| 827 | // goes, in one critical section; its proof waits for that record, |
| 828 | // which it may discharge (seeded pre-land review, P2). |
| 829 | if u.legacy && a.entries[host] != nil { |
| 830 | continue |
| 831 | } |
| 832 | u.busy = true |
| 833 | go a.reapUnprovenWake(host, u) |
| 834 | } |
| 835 | } |
| 836 | |
| 837 | // reapUnprovenWake is one retry of a retained failed wake. A leader that is |
| 838 | // still alive is stopped again, bounded. Once the generation is proven |
| 839 | // stopped, the replica is closed (or the lease released) and the retained |
| 840 | // wake is forgotten, which reopens the host to wakes. |
| 841 | // |
| 842 | // A failed close or release is logged and recorded in the stop history, and |
| 843 | // the wake is still forgotten: the generation, which is what retention |
| 844 | // guards, is proven stopped. A failed close is bounded by the lease's TTL, |
| 845 | // as for finishStop. A failed raw release is kept as an obligation that is |
| 846 | // retried with the same conditional delete (giveBack, #292). |
| 847 | // |
| 848 | // A retained legacy start (#292, Astra P1) has no process and no unit |
| 849 | // generation: it is proven by stopping its original run unit again |
| 850 | // (legacyRunGone), never by the generation proof. |
| 851 | func (a *Service) reapUnprovenWake(host string, u *unprovenWake) { |
| 852 | if u.proc != nil && !u.proc.Exited() { |
| 853 | ctx, cancel := context.WithTimeout(context.Background(), actStopTimeout) |
| 854 | err := u.proc.Stop(ctx) |
| 855 | cancel() |
| 856 | if err != nil && !errors.Is(err, ErrStillRunning) { |
| 857 | log.Printf("activator: %s: retry stop of failed wake's generation %d: %v", host, u.gen.N, err) |
| 858 | } |
| 859 | } |
| 860 | proven := false |
| 861 | if u.legacy { |
| 862 | proven = a.legacyRunGone(u.rt) == nil |
| 863 | } else { |
| 864 | proven = a.failedWakeStopped(u.rt, u.gen, u.proc, u.layout) |
| 865 | } |
| 866 | if !proven { |
| 867 | a.mu.Lock() |
| 868 | u.busy = false |
| 869 | a.mu.Unlock() |
| 870 | return |
| 871 | } |
| 872 | outcome := &stopObservation{mainStop: true, mainExited: true, uncertain: true} |
| 873 | ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) |
| 874 | var err error |
| 875 | if u.rep != nil { |
| 876 | err = u.rep.Close(ctx) |
| 877 | outcome.replicaClosed = err == nil |
| 878 | } else if u.lease != nil { |
| 879 | err = releaseLease(ctx, u.lease) |
| 880 | if err != nil { |
| 881 | a.oweRelease(host, u.lease, err) |
| 882 | } |
| 883 | } |
| 884 | cancel() |
| 885 | outcome.leaseReleased = err == nil |
| 886 | if err != nil { |
| 887 | log.Printf("activator: %s: failed wake's generation %d is proven stopped, but giving its lease back failed: %v", host, u.gen.N, err) |
| 888 | } else { |
| 889 | log.Printf("activator: %s: failed wake's generation %d is proven stopped; lease given back", host, u.gen.N) |
| 890 | } |
| 891 | a.mu.Lock() |
| 892 | if h := a.stopHistory[host]; u.legacy && h != nil && h.uncertain && h.owedBy == u { |
| 893 | // The history's only uncertainty was this start's, and #292's |
| 894 | // whole proof now holds: it is discharged, and the stages the |
| 895 | // failed wake never reached hold (seeded pre-land review, P2). |
| 896 | outcome.uncertain = false |
| 897 | outcome.sidecarStop, outcome.sidecarExited = true, true |
| 898 | if u.rep == nil { |
| 899 | outcome.replicaClosed = true |
| 900 | } |
| 901 | a.stopHistory[host] = outcome |
| 902 | } else { |
| 903 | a.rememberStopLocked(host, outcome) |
| 904 | } |
| 905 | if a.unprovenWakes[host] == u { |
| 906 | delete(a.unprovenWakes, host) |
| 907 | } |
| 908 | u.busy = false |
| 909 | a.mu.Unlock() |
| 910 | if _, listed := a.genIdentity(u.gen); listed || u.build != (release.StagedIdentity{}) { |
| 911 | // Proven gone: its build is no longer referenced. |
| 912 | _ = a.persistGenerations() |
| 913 | } |
| 914 | } |
| 915 | |
amadan