| index b5758edec..0225ac6b9 100644 |
| --- a/docs/superpowers/plans/2026-10-07-zero-downtime-deploys-3-swap.md |
| +++ b/docs/superpowers/plans/2026-10-07-zero-downtime-deploys-3-swap.md |
| @@ -1021,7 +1021,8 @@ git commit -m "zdd plan 3 task 1: swap types, the SwapBackend seam, flat staged |
| func (a *Service) genChildEmpty(rt registry.Route, m *genMember) bool |
| func (a *Service) genTerminated(rt registry.Route, g Gen, p Proc) bool // THE termination proof: leader exited and child empty; every lease- or set-giving path asks it |
| func (p *claimProcess) provenStopped(ctx context.Context) bool // claims-on: its own Stop completed (KillAndEmpty, exit, settle, close) |
| - func (a *Service) reapCurrentGen(e *actEntry) // a pending stop's current generation, proven empty before finishStop |
| + func (a *Service) reapCurrentGen(e *actEntry) // a pending stop finishes once releasable holds |
| + func (a *Service) releasable(e *actEntry) bool // THE retain-until-proven rule: no member, no stray sidecar, current generation genTerminated |
| type unprovenWake struct{ rt registry.Route; gen Gen; proc Proc; lease Lease; rep Rep; busy bool } |
| var errWakeUnproven error |
| func (a *Service) reapUnprovenWakesLocked() |
| @@ -1050,6 +1051,7 @@ import ( |
| "os/exec" |
| "slices" |
| "sync" |
| + "sync/atomic" |
| "testing" |
| "time" |
| |
| @@ -1301,6 +1303,70 @@ func TestMemberStopWithoutContainmentAcceptsTheLeaderExit(t *testing.T) { |
| } |
| } |
| |
| +// stubbornProc is a main process whose Stop fails with an error other than |
| +// ErrStillRunning (a claims-on KillAndEmpty error) and whose leader lives |
| +// on until exit. |
| +type stubbornProc struct{ exited atomic.Bool } |
| + |
| +func (p *stubbornProc) Stop(context.Context) error { return errors.New("cgroup.kill: the child is not empty") } |
| +func (p *stubbornProc) Exited() bool { return p.exited.Load() } |
| +func (p *stubbornProc) Crashed() bool { return false } |
| + |
| +// THE retain-until-proven rule (spec §7.4): a stop whose main process is |
| +// still alive after a failed kill keeps its lease. Only ErrStillRunning |
| +// kept it before; any other Stop error, with the leader alive, skipped the |
| +// proof (it ran only for an exited leader) and released the lease. |
| +func TestAStopKeepsTheLeaseWhileItsLeaderLivesOn(t *testing.T) { |
| + x := awakeForBarrier(t, true) |
| + p := &stubbornProc{} |
| + x.a.mu.Lock() |
| + x.a.entries[x.host].proc = p |
| + x.a.mu.Unlock() |
| + rep := x.b.lastRep() |
| + x.a.Bounce(x.host) |
| + waitFor(t, "the stop to pend", func() bool { |
| + x.a.mu.Lock() |
| + defer x.a.mu.Unlock() |
| + e := x.a.entries[x.host] |
| + return e != nil && e.pendingStop |
| + }) |
| + x.a.sweepOnce() |
| + time.Sleep(50 * time.Millisecond) |
| + if rep.isClosed() { |
| + t.Fatal("the lease was released while the leader still ran after a failed kill") |
| + } |
| + p.exited.Store(true) |
| + x.a.sweepOnce() |
| + waitFor(t, "the lease release", rep.isClosed) |
| +} |
| + |
| +// A stop keeps its lease while a stray sidecar of the host is tracked |
| +// (spec §7.4): its process may still run against the database. The lease |
| +// went with the stop before. |
| +func TestAStopKeepsTheLeaseWhileAStraySidecarRuns(t *testing.T) { |
| + x := awakeForBarrier(t, true) |
| + stray := &fakeProc{label: "sidecar", rec: x.b, stuck: true} |
| + x.a.mu.Lock() |
| + x.a.straySidecars = map[string][]Proc{x.host: {stray}} |
| + x.a.mu.Unlock() |
| + rep := x.b.lastRep() |
| + x.a.Bounce(x.host) |
| + waitFor(t, "the stop to pend", func() bool { |
| + x.a.mu.Lock() |
| + defer x.a.mu.Unlock() |
| + e := x.a.entries[x.host] |
| + return e != nil && e.pendingStop |
| + }) |
| + if rep.isClosed() { |
| + t.Fatal("the lease was released while a stray sidecar ran") |
| + } |
| + x.a.mu.Lock() |
| + delete(x.a.straySidecars, x.host) // proven gone (Task 7's reaper does this) |
| + x.a.mu.Unlock() |
| + x.a.sweepOnce() |
| + waitFor(t, "the lease release", rep.isClosed) |
| +} |
| + |
| // failingContainment is a claims-on child whose KillAndEmpty fails until |
| // heal is called. |
| type failingContainment struct { |
| @@ -1622,6 +1688,11 @@ func (a *Service) stopMember(e *actEntry, m *genMember) bool { |
| // process never ran. |
| func (a *Service) genTerminated(rt registry.Route, g Gen, p Proc) bool { |
| if p == nil { |
| + if g.Worker { |
| + // A worker's host cgroup is proven whether or not a process |
| + // handle is left: a dead leader's descendants live there (Task 11). |
| + return a.genChildEmpty(rt, &genMember{gen: g}) |
| + } |
| return true |
| } |
| if !p.Exited() { |
| @@ -1769,10 +1840,7 @@ func (a *Service) reapExited(e *actEntry, m *genMember) { |
| // the current generation's child is proven empty. A failed proof leaves the |
| // stop pending, and the next sweep tries again. |
| func (a *Service) reapCurrentGen(e *actEntry) { |
| - a.mu.Lock() |
| - rt, g, p := e.route, e.gen, e.proc |
| - a.mu.Unlock() |
| - if !a.genTerminated(rt, g, p) { |
| + if !a.releasable(e) { |
| a.mu.Lock() |
| e.genReaping = false |
| a.mu.Unlock() |
| @@ -1780,11 +1848,15 @@ func (a *Service) reapCurrentGen(e *actEntry) { |
| } |
| a.mu.Lock() |
| e.genReaping, e.pendingStop = false, false |
| - outcome := e.stopOutcome |
| + outcome, p := e.stopOutcome, e.proc |
| a.mu.Unlock() |
| if outcome == nil { |
| outcome = &stopObservation{} |
| } |
| + if p == nil { |
| + a.finishStop(e, outcome) // only the barrier left it pending |
| + return |
| + } |
| a.finishPendingStop(e, outcome) |
| } |
| |
| @@ -1899,27 +1971,47 @@ At the end of `stop()`, replace `a.finishStop(e, outcome)` (line 2268) with: |
| a.mu.Unlock() |
| return |
| } |
| - if a.gens && e.proc != nil && e.proc.Exited() { |
| - // The current generation is a generation like any other (spec |
| - // §7.4): its leader's exit is not its termination until its child |
| - // is empty. Unproven, the stop stays pending with the lease held, |
| - // exactly as for ErrStillRunning, and the sweep retries the proof. |
| + if a.gens && !a.releasable(e) { // swapsim:M8 |
| + // THE retain-until-proven rule (spec §7.4): unproven, the stop stays |
| + // pending with the lease held, exactly as for ErrStillRunning, and |
| + // the sweep retries (reapCurrentGen). That covers a leader still |
| + // alive after a failed kill (a claims-on KillAndEmpty error is not |
| + // ErrStillRunning, so it reaches here), a child not proven empty, a |
| + // worker's host cgroup, and a stray sidecar. |
| + log.Printf("activator: stop %s: not every process of the entry is proven gone; keeping the lease until it is", e.host) |
| a.mu.Lock() |
| - rt, g := e.route, e.gen |
| + e.pendingStop = true |
| + e.stopOutcome = outcome |
| + e.stopRetriedAt = a.now() |
| a.mu.Unlock() |
| - if !a.genTerminated(rt, g, e.proc) { // swapsim:M8 |
| - log.Printf("activator: stop %s: generation %d's child is not proven empty; keeping the lease until it is", e.host, g.N) |
| - a.mu.Lock() |
| - e.pendingStop = true |
| - e.stopOutcome = outcome |
| - e.stopRetriedAt = a.now() |
| - a.mu.Unlock() |
| - return |
| - } |
| + return |
| } |
| a.finishStop(e, outcome) |
| ``` |
| |
| +Add to `opset.go`: |
| + |
| +```go |
| +// releasable is THE retain-until-proven rule for an entry's lease (spec |
| +// §7.4): a stop finishes, and its lease is released, only when every |
| +// member has left the operation set, no stray sidecar of the host is |
| +// tracked (Task 7), and the current generation has passed genTerminated |
| +// (its leader exited and its child, or a worker's host cgroup, empty). It |
| +// runs without a.mu held; genTerminated may take DeadlineControl. |
| +func (a *Service) releasable(e *actEntry) bool { |
| + a.mu.Lock() |
| + members, strays := len(e.ops.members), len(a.straySidecars[e.host]) |
| + rt, g, p := e.route, e.gen, e.proc |
| + a.mu.Unlock() |
| + if members > 0 || strays > 0 { |
| + return false |
| + } |
| + return a.genTerminated(rt, g, p) |
| +} |
| +``` |
| + |
| +Add the field it reads to `Service` now: `straySidecars map[string][]Proc // legacy sidecars whose stop is unproven, per host; Task 7 tracks into it; under a.mu`. |
| + |
| Add to `claimwake.go`, beside `claimProcess.Stop`: |
| |
| ```go |
| @@ -1956,6 +2048,17 @@ with |
| if !e.pendingStop || len(e.ops.members) > 0 { |
| continue |
| } |
| + if a.gens && (e.proc == nil || e.proc.Exited()) { |
| + // Members gone and the main process (if any) exited: the stop |
| + // finishes once the retain-until-proven rule holds (releasable, |
| + // in reapCurrentGen). A process still alive falls through to |
| + // main's re-stop cadence below. |
| + if !e.genReaping { |
| + e.genReaping = true |
| + go a.reapCurrentGen(e) |
| + } |
| + continue |
| + } |
| if e.proc == nil { |
| // Only the barrier leaves a stop pending without a process. Its |
| // members are gone now, so the stop can finish. |
| @@ -1967,15 +2070,6 @@ with |
| go a.finishStop(e, outcome) |
| continue |
| } |
| - if a.gens && e.proc.Exited() { |
| - // The main process has exited: its generation's child must be |
| - // proven empty before the stop finishes (spec §7.4). |
| - if !e.genReaping { |
| - e.genReaping = true |
| - go a.reapCurrentGen(e) |
| - } |
| - continue |
| - } |
| ``` |
| |
| and add, as the first statement of `finishPendingStops` (before the loop), `a.reapUnprovenWakesLocked()`. |
| @@ -2405,6 +2499,8 @@ git commit -m "zdd plan 3 task 3: swap admission through hostwrite, claims off a |
| func (a *Service) genHealthy(ctx context.Context, g Gen) error |
| type stepOutcome int; const stepOK, stepCancelled, stepStale, stepFailed stepOutcome |
| func classifyStep(lifecycle context.Context, err error) stepOutcome // THE outcome classifier: only stepFailed is a failure of the build |
| + type gateVerdict int; const gateProceed, gateSuperseded, gateNoOwnership gateVerdict |
| + func verdictOf(err error) gateVerdict // the attempt gate's typed verdict; NoOwnership (lease, fence, shutdown, a claims refusal such as hostguard.ErrStampMismatch) stops every spawn, restores included |
| func stepResult(lifecycle context.Context, err error) (SwapResult, bool) |
| // Service gains: genHealth func(ctx context.Context, g Gen) error (test seam) |
| var errSwapOff, errSwapBusy, errSwapKind, errSwapStopped, errSuccessorExited, errSwapMode error |
| @@ -2691,6 +2787,7 @@ import ( |
| "time" |
| |
| "github.com/carlosframework/platform/internal/cgroup" |
| + "github.com/carlosframework/platform/internal/hostguard" |
| "github.com/carlosframework/platform/internal/hostwrite" |
| "github.com/carlosframework/platform/internal/registry" |
| ) |
| @@ -2945,6 +3042,10 @@ func TestClassifyStep(t *testing.T) { |
| {"the barrier", live, errSwapStopped, stepCancelled}, |
| {"superseded", live, fmt.Errorf("pin: %w", errSwapSuperseded), stepStale}, |
| {"lease lost", live, errSwapLeaseLost, stepStale}, |
| + // Claims validation at the exec (spawnClaimedGen's Work.With): |
| + // nothing ran, so no failure of the build and no halted cohort. |
| + {"a changed stamp at the exec", live, fmt.Errorf("with: %w", hostguard.ErrStampMismatch), stepStale}, |
| + {"a durable fence at the exec", live, fmt.Errorf("with: %w", hostguard.ErrFenced), stepStale}, |
| {"a health deadline", live, fmt.Errorf("health: %w", context.DeadlineExceeded), stepFailed}, |
| {"HTTP 500", live, errors.New("GET / answered 500"), stepFailed}, |
| } { |
| @@ -2954,6 +3055,30 @@ func TestClassifyStep(t *testing.T) { |
| } |
| } |
| |
| +// The gate's verdict is typed (spec §6.3, §7.4): a supersession may restore |
| +// behind a fresh ownership check; a lost lease, a fence, shutdown or a |
| +// claims refusal stops every spawn. |
| +func TestVerdictOf(t *testing.T) { |
| + for _, tc := range []struct { |
| + err error |
| + want gateVerdict |
| + }{ |
| + {nil, gateProceed}, |
| + {errors.New("GET / answered 500"), gateProceed}, |
| + {fmt.Errorf("gate: %w", errSwapSuperseded), gateSuperseded}, |
| + {errSwapLeaseLost, gateNoOwnership}, |
| + {errSwapFenced, gateNoOwnership}, |
| + {errSwapShutdown, gateNoOwnership}, |
| + {errSwapNotAwake, gateNoOwnership}, |
| + {fmt.Errorf("with: %w", hostguard.ErrStampMismatch), gateNoOwnership}, |
| + {fmt.Errorf("with: %w", hostguard.ErrFenced), gateNoOwnership}, |
| + } { |
| + if got := verdictOf(tc.err); got != tc.want { |
| + t.Errorf("verdictOf(%v) = %d, want %d", tc.err, got, tc.want) |
| + } |
| + } |
| +} |
| + |
| // Spec §6.3, §7.4: the checks run again at the exec, not only at the |
| // admission. A newer attempt that lands while the parent is being raised |
| // (or, with plan 5, while the pins are being published) means nothing |
| @@ -3037,7 +3162,7 @@ func TestSwapClaimsOn(t *testing.T) { |
| |
| - [ ] **Step 3: Run the tests to verify they fail** |
| |
| -Run: `TMPDIR=/tmp/k nice -n 19 go test ./internal/activator -run 'TestSwapOverlap|TestSwapRefusals|TestSwapWitnessNoAdmission|TestSwapClaimsOn' -count=1` |
| +Run: `TMPDIR=/tmp/k nice -n 19 go test ./internal/activator -run 'TestSwapOverlap|TestSwapRefusals|TestSwapWitnessNoAdmission|TestSwapClaimsOn|TestClassifyStep|TestVerdictOf' -count=1` |
| Expected: build failure: `f.a.Swap undefined`, `undefined: errSwapOff`, `unknown field genHealth`. |
| |
| - [ ] **Step 4: Write the code** |
| @@ -3051,7 +3176,7 @@ Add to `Service` in `activator.go` (after `OnWake`, line 682): |
| genHealth func(ctx context.Context, g Gen) error |
| ``` |
| |
| -Append to `internal/activator/swap.go`, and add its imports (`context`, `errors`, `fmt`, `time`, `internal/cgroup` and `internal/registry`): |
| +Append to `internal/activator/swap.go`, and add its imports (`context`, `errors`, `fmt`, `time`, `internal/cgroup`, `internal/hostbox`, `internal/hostguard` and `internal/registry`; neither `hostbox` nor `hostguard` imports `activator`): |
| |
| ```go |
| var ( |
| @@ -3091,13 +3216,52 @@ func classifyStep(lifecycle context.Context, err error) stepOutcome { |
| case lifecycle != nil && lifecycle.Err() != nil, |
| errors.Is(err, context.Canceled), errors.Is(err, errSwapStopped), errors.Is(err, ErrFenced): |
| return stepCancelled |
| - case errors.Is(err, errSwapSuperseded), errors.Is(err, errSwapNotAwake), errors.Is(err, errSwapLeaseLost), |
| - errors.Is(err, errSwapFenced), errors.Is(err, errSwapShutdown): |
| + case verdictOf(err) != gateProceed: |
| + // The attempt gate's own refusal, wherever it surfaced: at Step 3a, |
| + // or as a claims validation inside the work at the exec. Nothing of |
| + // the build ran because of it (spec §7.4). |
| return stepStale |
| } |
| return stepFailed |
| } |
| |
| +// gateVerdict is the attempt gate's answer, typed (spec §6.3, §7.4). |
| +type gateVerdict int |
| + |
| +const ( |
| + // gateProceed: the attempt is current and owns the host. |
| + gateProceed gateVerdict = iota |
| + // gateSuperseded: still the owner, but a newer attempt or a halted |
| + // cohort replaced this one. A restore of what it replaced may run, and |
| + // only behind a fresh ownership check at that exec. |
| + gateSuperseded |
| + // gateNoOwnership: the entry is not current, the lease is lost, a fence |
| + // or the agent's shutdown holds the host, or the claims validation |
| + // refused (a changed stamp, a durable fence, a changed claim). Nothing |
| + // may spawn, a restore included. |
| + gateNoOwnership |
| +) |
| + |
| +// verdictOf types an attempt-gate error. An error that is not the gate's |
| +// (a failure of the build) is gateProceed: the gate did not refuse. |
| +func verdictOf(err error) gateVerdict { |
| + switch { |
| + case err == nil: |
| + return gateProceed |
| + case errors.Is(err, errSwapSuperseded): |
| + return gateSuperseded |
| + case errors.Is(err, errSwapNotAwake), errors.Is(err, errSwapLeaseLost), errors.Is(err, errSwapFenced), |
| + errors.Is(err, errSwapShutdown), errors.Is(err, ErrFenced), |
| + errors.Is(err, hostguard.ErrStampMismatch), errors.Is(err, hostguard.ErrFenced), |
| + errors.Is(err, hostguard.ErrInactive), errors.Is(err, hostguard.ErrRetiredStamp), |
| + errors.Is(err, hostguard.ErrOldStamp), errors.Is(err, hostguard.ErrClaimIncarnation), |
| + errors.Is(err, hostguard.ErrRecordIncarnation), errors.Is(err, hostguard.ErrNotParticipant), |
| + errors.Is(err, hostbox.ErrChanged): |
| + return gateNoOwnership |
| + } |
| + return gateProceed |
| +} |
| + |
| // stepResult is classifyStep for a swap step: a cancelled or stale step |
| // abandons the attempt (over is true); a failure of the build is the |
| // caller's to report (over is false). |
| @@ -5249,7 +5413,7 @@ func (a *Service) reapStraySidecarsLocked() { |
| } |
| ``` |
| |
| -Add to `Service`: `straySidecars map[string][]Proc // legacy sidecars whose stop is unproven, per host; under a.mu`. Call `a.reapStraySidecarsLocked()` beside Task 2's `a.reapUnprovenWakesLocked()` at the top of `finishPendingStops`. In `doWake`, extend Task 2's unproven-wake refusal so a tracked stray also refuses: `_, unproven := a.unprovenWakes[host]; unproven = unproven || len(a.straySidecars[host]) > 0`. |
| +(`Service.straySidecars` is Task 2's, and Task 2's `releasable` already keeps a stop's lease while the host has one.) Call `a.reapStraySidecarsLocked()` beside Task 2's `a.reapUnprovenWakesLocked()` at the top of `finishPendingStops`. In `doWake`, extend Task 2's unproven-wake refusal so a tracked stray also refuses: `_, unproven := a.unprovenWakes[host]; unproven = unproven || len(a.straySidecars[host]) > 0`. |
| |
| In `stop()`'s co-stop, with generations on, a sidecar whose stop is unproven is tracked as well. After main's `outcome.sidecarStop, outcome.sidecarExited = err == nil, e.sidecar.Exited()`, add: |
| |
| @@ -6584,7 +6748,10 @@ Claims-on wakes spawn through the claim runtime's provider (`ClaimStart.Build`), |
| func (b *SplitBackend) GenPaths(rt registry.Route, n uint64) (Gen, bool) |
| func (a *Service) claimGenAt(rt registry.Route, n uint64) Gen |
| func (a *Service) entryParent(e *actEntry, s ClaimStart) (*cgroup.Prepared, error) |
| - func (a *Service) closeEntryParent(e *actEntry) |
| + func (a *Service) closeEntryParent(e *actEntry) // Close (not CloseRemove) the empty parent; a handle that does not close is kept and retried |
| + var closeStrictParent = (*cgroup.Prepared).Close |
| + func (a *Service) retryUnclosedParentsLocked() |
| + // Service gains unclosedParents []*cgroup.Prepared; parentsRetrying bool |
| func (a *Service) setParent(ctx context.Context, gb SwapBackend, e *actEntry, rt registry.Route, cur cgroup.Limit, succ *cgroup.Limit) error // captured tiers only |
| func (a *Service) spawnSuccessor(ctx context.Context, gb SwapBackend, e *actEntry, rt registry.Route, work hostwrite.Work, g Gen, gate func() error) (Gen, Proc, error) |
| func (a *Service) startClaimedSuccessor(ctx context.Context, e *actEntry, rt registry.Route, work hostwrite.Work, g Gen, gate func() error) (Gen, Proc, error) // gate after Prepare, and as spawnClaimedGen's before hook |
| @@ -6760,6 +6927,66 @@ func TestStartAdmittedFallsBackToTheGoodBuildOnce(t *testing.T) { |
| } |
| } |
| |
| +// Spec §7.3, §7.4: an entry's empty strict parent is closed with Close, |
| +// which releases its reservation; a handle that does not close is kept and |
| +// retried, never dropped. CloseRemove (a generation child's) refused every |
| +// parent, leaking the reservation so the host's next wake was refused. |
| +func TestAnEmptyStrictParentIsClosedAndKeptUntilItCloses(t *testing.T) { |
| + old := closeStrictParent |
| + t.Cleanup(func() { closeStrictParent = old }) |
| + var calls atomic.Int32 |
| + var failing atomic.Bool |
| + failing.Store(true) |
| + closeStrictParent = func(*cgroup.Prepared) error { |
| + calls.Add(1) |
| + if failing.Load() { |
| + return errors.New("strict: still populated") |
| + } |
| + return nil |
| + } |
| + a, _, _, _ := testActivator(t, time.Hour) |
| + e := &actEntry{host: "x.example"} |
| + e.ops.parent = &cgroup.Prepared{} |
| + a.closeEntryParent(e) |
| + if calls.Load() != 1 { |
| + t.Fatalf("Close calls = %d, want the parent closed with Close", calls.Load()) |
| + } |
| + a.mu.Lock() |
| + kept := len(a.unclosedParents) |
| + a.mu.Unlock() |
| + if kept != 1 { |
| + t.Fatal("a parent that did not close was dropped") |
| + } |
| + failing.Store(false) |
| + a.sweepOnce() |
| + waitFor(t, "the retried close", func() bool { |
| + a.mu.Lock() |
| + defer a.mu.Unlock() |
| + return len(a.unclosedParents) == 0 && !a.parentsRetrying |
| + }) |
| + if calls.Load() != 2 { |
| + t.Fatalf("Close calls = %d, want one retry", calls.Load()) |
| + } |
| +} |
| + |
| +// Spec §7.4: the claims validation inside spawnClaimedGen's Work.With (a |
| +// changed stamp, a durable fence) refuses the exec. Nothing ran, so the |
| +// attempt is abandoned, never SwapFailed: a failure record would halt the |
| +// cohort for a build that never executed. |
| +func TestAClaimsRefusalAtTheExecIsNoFailure(t *testing.T) { |
| + for _, refusal := range []error{hostguard.ErrStampMismatch, hostguard.ErrFenced} { |
| + f := newSwapFixture(t) |
| + f.a.cfg.HostWrites = hostwrite.Config{Enabled: true, Selector: &swapSelector{work: &swapWork{}}} |
| + f.a.cfg.Claims = &ClaimRuntime{Pins: true, Generations: true} |
| + f.a.claimSpawn = func(context.Context, *actEntry, registry.Route, hostwrite.Work, Gen) (Gen, Proc, error) { |
| + return Gen{}, nil, fmt.Errorf("work: %w", refusal) |
| + } |
| + if res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest); res.Outcome != SwapAbandoned { |
| + t.Fatalf("%v at the exec: Swap = %+v, want abandoned", refusal, res) |
| + } |
| + } |
| +} |
| + |
| // claimedStartFor is a ClaimStart for obs with every hook a no-op, version v. |
| func claimedStartFor(obs hostclaim.Observed, work hostwrite.Work, v string) ClaimStart { |
| return ClaimStart{Host: obs.Host, Work: work, Identity: hostwrite.Identity{Config: "c", Release: "r"}, Version: v, |
| @@ -6888,11 +7115,11 @@ func TestClaimsOnSuccessorReChecksAfterPrepare(t *testing.T) { |
| } |
| ``` |
| |
| -Add `"sync"` and `"github.com/carlosframework/platform/internal/cgroup"` to the file's imports. |
| +Add `"fmt"`, `"sync"`, `"sync/atomic"`, `"github.com/carlosframework/platform/internal/cgroup"` and `"github.com/carlosframework/platform/internal/hostguard"` to the file's imports. |
| |
| - [ ] **Step 2: Run the tests to verify they fail** |
| |
| -Run: `TMPDIR=/tmp/k nice -n 19 go test ./internal/activator -run 'TestClaimsOnSwapNeedsGenerations|TestStartAdmittedRefusesAGenerationMismatch|TestClaimGenAt|TestStartAdmittedFallsBack|TestClaimsOnSuccessorReChecks|TestAClaimsOnHoldRunsTheGate|TestAClaimedWakeCancelled' -count=1` |
| +Run: `TMPDIR=/tmp/k nice -n 19 go test ./internal/activator -run 'TestClaimsOnSwapNeedsGenerations|TestStartAdmittedRefusesAGenerationMismatch|TestClaimGenAt|TestStartAdmittedFallsBack|TestClaimsOnSuccessorReChecks|TestAClaimsOnHoldRunsTheGate|TestAClaimedWakeCancelled|TestAClaimsRefusalAtTheExec|TestAnEmptyStrictParent' -count=1` |
| Expected: build failure: `undefined: errClaimsNoGenerations`, `unknown field Gen in struct literal of type ClaimStart`, `a.claimGenAt undefined`, `unknown field Fallback in struct literal of type StartPolicy`. |
| |
| - [ ] **Step 3: Extend the claim types and `startAdmitted`** |
| @@ -7227,9 +7454,18 @@ func (a *Service) entryParent(e *actEntry, s ClaimStart) (*cgroup.Prepared, erro |
| return p, nil |
| } |
| |
| +// closeStrictParent closes an entry's empty strict parent: Close, which |
| +// releases its descriptors and its reservation. CloseRemove is for a |
| +// generation child and refuses a parent handle (plan 2 Task 3), which |
| +// leaked the reservation and refused the host's next wake. A var so a test |
| +// can make it fail. |
| +var closeStrictParent = (*cgroup.Prepared).Close |
| + |
| // closeEntryParent releases the entry's strict parent. finishStop calls it |
| // after the barrier has joined every generation, and wake() after a failed |
| -// claims-on wake. |
| +// claims-on wake. A handle that does not close (still populated, an |
| +// identity check failed) is kept, never dropped: it holds the host's |
| +// reservation, and the sweep retries it (retryUnclosedParentsLocked). |
| func (a *Service) closeEntryParent(e *actEntry) { |
| a.mu.Lock() |
| p := e.ops.parent |
| @@ -7238,9 +7474,35 @@ func (a *Service) closeEntryParent(e *actEntry) { |
| if p == nil { |
| return |
| } |
| - if err := p.CloseRemove(); err != nil { |
| - log.Printf("activator: release the cgroup parent of %s: %v", e.host, err) |
| + if err := closeStrictParent(p); err != nil { |
| + log.Printf("activator: release the cgroup parent of %s: %v (kept; the sweep retries)", e.host, err) |
| + a.mu.Lock() |
| + a.unclosedParents = append(a.unclosedParents, p) |
| + a.mu.Unlock() |
| + } |
| +} |
| + |
| +// retryUnclosedParentsLocked closes the strict parents a stop could not, |
| +// keeping each until it closes. The caller holds a.mu; the closes run on |
| +// their own goroutine. |
| +func (a *Service) retryUnclosedParentsLocked() { |
| + if len(a.unclosedParents) == 0 || a.parentsRetrying { |
| + return |
| } |
| + pending := a.unclosedParents |
| + a.unclosedParents, a.parentsRetrying = nil, true |
| + go func() { |
| + var still []*cgroup.Prepared |
| + for _, p := range pending { |
| + if err := closeStrictParent(p); err != nil { |
| + still = append(still, p) |
| + } |
| + } |
| + a.mu.Lock() |
| + a.unclosedParents = append(a.unclosedParents, still...) |
| + a.parentsRetrying = false |
| + a.mu.Unlock() |
| + }() |
| } |
| |
| // setParent raises the parent to the incumbent's tier plus the successor's |
| @@ -7391,6 +7653,8 @@ In `leaveSet` (`opset.go`), the restore block becomes: |
| } |
| ``` |
| |
| +Add to `Service`: `unclosedParents []*cgroup.Prepared; parentsRetrying bool // strict parents a stop could not close, kept and retried; under a.mu`. Call `a.retryUnclosedParentsLocked()` beside Task 2's `a.reapUnprovenWakesLocked()` at the top of `finishPendingStops`. |
| + |
| In `finishStop` (`activator.go`), insert at the top, before `ctx, cancel := context.WithTimeout(...)`: |
| |
| ```go |
| @@ -7470,10 +7734,12 @@ package activator |
| import ( |
| "context" |
| "errors" |
| + "path/filepath" |
| "sync" |
| "testing" |
| "time" |
| |
| + "github.com/carlosframework/platform/internal/provision" |
| "github.com/carlosframework/platform/internal/registry" |
| ) |
| |
| @@ -7512,6 +7778,23 @@ func TestWorkerGenerations(t *testing.T) { |
| } |
| } |
| |
| +// StartWorker spawns every worker into the flat host cgroup, whatever the |
| +// row's layout, so KillHost proves that cgroup for a nested-looking row too. |
| +// Here the cgroup is a fixture, not cgroup v2, so the proof is refused. |
| +// Answering nil for a nested layout skipped the proof for a worker that |
| +// runs in carlos-exec@<host> all the same. |
| +func TestKillHostProvesANestedLayoutWorkersHostCgroup(t *testing.T) { |
| + host := "nw.example.com" |
| + b, _ := delegatedBackend(t, host) |
| + rd := t.TempDir() |
| + b.RunDirOverride = rd |
| + rt := registry.Route{Host: host, Kind: "sidecar", DBPath: provision.NestedDBPath(b.DataDir, host), |
| + Socket: filepath.Join(rd, "instances", provision.InstanceID(host), "app.sock")} |
| + if err := b.KillHost(context.Background(), rt); err == nil { |
| + t.Fatal("KillHost answered a nested-layout worker's host cgroup as proven without proving it") |
| + } |
| +} |
| + |
| // hostKillBackend is fakeBackend plus KillHost, failing while bad. |
| type hostKillBackend struct { |
| *fakeBackend |
| @@ -7567,6 +7850,65 @@ func TestWorkerTerminationProvesItsHostCgroup(t *testing.T) { |
| a.sweepOnce() |
| waitFor(t, "the lease release", rep.isClosed) |
| }) |
| + t.Run("a respawn with no old handle still proves the host cgroup", func(t *testing.T) { |
| + a, b, _, _ := testActivator(t, time.Hour) |
| + a.gens = true |
| + hk := &hostKillBackend{fakeBackend: b} |
| + a.backend = hk |
| + rt := testWorkerRoute("bot3.example.com") |
| + if err := a.WakeWorker(rt); err != nil { |
| + t.Fatal(err) |
| + } |
| + hk.hmu.Lock() |
| + hk.bad = true |
| + hk.hmu.Unlock() |
| + a.mu.Lock() |
| + // main's signature; once plan 4 Task 9 has merged, pass nil as its |
| + // fourth (onto) argument. |
| + a.respawnWorkerLocked(a.entries[rt.Host], nil, "test: a restart with no old handle") |
| + a.mu.Unlock() |
| + waitFor(t, "the respawn to finish", func() bool { return !a.entryRespawning(rt.Host) }) |
| + if got := b.workerStarts(); got != 1 { |
| + t.Fatalf("worker starts = %d: a nil-stopOld respawn skipped the host cgroup proof", got) |
| + } |
| + }) |
| + t.Run("a failed worker wake keeps its lease", func(t *testing.T) { |
| + old := sidecarSettle |
| + sidecarSettle = 20 * time.Millisecond |
| + t.Cleanup(func() { sidecarSettle = old }) |
| + a, b, _, _ := testActivator(t, time.Hour) |
| + a.gens = true |
| + hk := &hostKillBackend{fakeBackend: b, bad: true} |
| + a.backend = hk |
| + b.mu.Lock() |
| + b.workerDeadOnStart = true // the wake fails its settle |
| + b.mu.Unlock() |
| + if err := a.WakeWorker(testWorkerRoute("bot4.example.com")); err == nil { |
| + t.Fatal("a worker dead on start woke") |
| + } |
| + // The lease goes back by Release, or by closing the replica when the |
| + // wake had one (cleanupFailedWake). |
| + given := func() bool { |
| + b.mu.Lock() |
| + released, reps := b.releases, append([]*fakeRep(nil), b.reps...) |
| + b.mu.Unlock() |
| + if released > 0 { |
| + return true |
| + } |
| + for _, r := range reps { |
| + if r.isClosed() { |
| + return true |
| + } |
| + } |
| + return false |
| + } |
| + if given() { |
| + t.Fatal("a failed worker wake gave its lease back over an unproven host cgroup") |
| + } |
| + hk.heal() |
| + a.sweepOnce() |
| + waitFor(t, "the lease given back", given) |
| + }) |
| t.Run("a respawn execs nothing beside the old child", func(t *testing.T) { |
| old := sidecarSettle |
| sidecarSettle = 20 * time.Millisecond |
| @@ -7594,7 +7936,7 @@ func TestWorkerTerminationProvesItsHostCgroup(t *testing.T) { |
| |
| - [ ] **Step 2: Run the test to verify it fails** |
| |
| -Run: `TMPDIR=/tmp/k nice -n 19 go test ./internal/activator -run 'TestWorkerGenerations|TestWorkerTermination' -count=1` |
| +Run: `TMPDIR=/tmp/k nice -n 19 go test ./internal/activator -run 'TestWorkerGenerations|TestWorkerTermination|TestKillHostProvesANestedLayout' -count=1` |
| Expected: FAIL: `gens true: worker generation {N:0 ...}`. |
| |
| - [ ] **Step 3: Write the code** |
| @@ -7620,6 +7962,18 @@ Add the field to plan 1's `Gen`, after Task 1's `Tier`: |
| Worker bool |
| ``` |
| |
| +A failed worker wake carries its Worker generation too, so `cleanupFailedWake` (Task 2) proves its host cgroup before the lease goes. In `doWake`'s worker branch, directly after the spawn line that sets `proc` (plan 4's `startWorker` call, or plan 5's `spawnPlannedWorker`) and before any error handling, add: |
| + |
| +```go |
| + if a.gens { |
| + // The wake's generation, whether or not the worker becomes ready |
| + // (zero-downtime spec §7.4): cleanupFailedWake proves it. |
| + w.gen = Gen{N: nextGen(), Build: ident.Version, Worker: true} |
| + } |
| +``` |
| + |
| +and make the `wake()` block above keep a number already set: `if a.gens && e.worker && e.gen.N == 0 {` stays as the fallback for a worker woken some other way. |
| + |
| In `opset.go`'s `genChildEmpty`, insert directly after its `claimProcess` branch: |
| |
| ```go |
| @@ -7656,13 +8010,11 @@ Add to `genbackend.go`: |
| ```go |
| // KillHost kills whatever is left in rt's own host cgroup and returns once |
| // it is empty: a worker's termination proof (zero-downtime spec §7.4, §7.9). |
| -// A nested or unit-backed worker is a systemd unit, whose stop already |
| -// takes its whole cgroup, so there is nothing more to prove here. An absent |
| -// cgroup is empty. No cgroup delegation at all is an unproven kill. |
| +// It does not look at the row's layout: StartWorker spawns every worker |
| +// through spawnInCgroup into carlos-exec@<host>, whatever the layout says, |
| +// so a nested-looking worker is proven there too. An absent cgroup is |
| +// empty. No cgroup delegation at all is an unproven kill. |
| func (b *ExecBackend) KillHost(ctx context.Context, rt registry.Route) error { |
| - if p := b.pathsFor(rt); p.layout == LayoutNested { |
| - return nil |
| - } |
| if b.Cgroups == nil { |
| return cgroup.ErrNoDelegation |
| } |
| @@ -7738,7 +8090,12 @@ Directly after the retire block (`if stopOld != nil { … }`) and before the "St |
| // spec §7.4, §7.9). Unproven: nothing spawns; the entry keeps its |
| // lease, and the sweep's crash path stops it through genTerminated. |
| retired := true |
| - if a.gens && stopOld != nil { |
| + if a.gens { |
| + // stopOld may be nil (a crash restart, a restore): the old child |
| + // is gone, but what it forked lives in the host cgroup, and |
| + // genTerminated proves a Worker generation there with or |
| + // without a handle. |
| + oldGen.Worker = true |
| retired = a.genTerminated(rt, oldGen, stopOld) |
| } |
| ``` |
| @@ -8653,7 +9010,7 @@ M4|opset.go|swapsim:M4|s/proven := a\.stopMember\(e, m\)/proven := true; go a.st |
| M5|swapadmit.go|swapsim:M5|s/!waking \&\& e\.state != actAwake/false/ |
| M6|activator.go|swapsim:M6|s/if a\.gens \{/if false {/ |
| M7|opset.go|swapsim:M7|s/return a\.genTerminated\(rt, m\.gen, m\.proc\)/return true || a.genTerminated(rt, m.gen, m.proc)/ |
| -M8|activator.go|swapsim:M8|s/if !a\.genTerminated\(/if false \&\& !a.genTerminated(/' |
| +M8|activator.go|swapsim:M8|s/if a\.gens \&\& !a\.releasable\(e\)/if false \&\& !a.releasable(e)/' |
| fail=0 |
| while IFS='|' read -r name file marker sub; do |
| src="$SCRATCH/tree/$A/$file" |
amadan