amadan
activator: seeds and flag-off stops prove their processes gone before the lease goes
Flag off, a flat or foreign exec row's terminal stop proves carlos-exec@<host> empty before finishStop releases the lease (stopHostProven): EmptyHost reads cgroup.events and kills only when populated, ErrNoDelegation keeps the leader-only proof, and a failed proof keeps the stop pending with the lease; the pending path retries it through reapUncontainedStop. A sidecar start still in flight past stop()'s bounded wait keeps the stop pending until it returns; the restart stops the sidecar it brought back before closing its done channel, and only then does the host proof run. actEntry.launchLayout is set at install so nested and unit rows skip the proof. Nothing new runs under a.mu.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
4 files changed,
+491
−64
internal/activator/activator.go+115 −26internal/activator/activator_test.go+251 −38internal/activator/genbackend.go+43 −0internal/activator/opset.go+82 −0
diff --git a/internal/activator/activator.go b/internal/activator/activator.go| index c4f42dda4..5279718d8 100644 |
| --- a/internal/activator/activator.go |
| +++ b/internal/activator/activator.go |
| @@ -508,10 +508,21 @@ type actEntry struct { |
| // exited, so an intentional stop is never read as a crash (spec §6.4). |
| // Written under a.mu. |
| settleCancel context.CancelFunc |
| - // genReaping is true while reapCurrentGen is running for this pending |
| - // stop, so finishPendingStops does not start a second one. Written |
| - // under a.mu. |
| + // genReaping is true while reapCurrentGen (or, flag off, |
| + // reapUncontainedStop) is running for this pending stop, so |
| + // finishPendingStops does not start a second one. Written under a.mu. |
| genReaping bool |
| + // sidecarJoin is a sidecar start a flag-off terminal stop stopped |
| + // waiting for (sidecarStartJoinWait) while it was still in flight: the |
| + // stop stays pending, with the lease, until it closes (Task 8, F6 plan |
| + // 5), and only then is the host cgroup proven empty. Written under a.mu. |
| + sidecarJoin chan struct{} |
| + // launchLayout is the layout the wake launched this entry's child on |
| + // (wokenInstance.launchLayout), set at install. A flag-off terminal stop |
| + // decides from it whether the host cgroup must be proven empty |
| + // (stopHostProven, hostCgroupIsTheWakes). "" requires the proof. Written |
| + // under a.mu. |
| + launchLayout Layout |
| // The rollout scheduler's view of this entry (zero-downtime plan 5). |
| // plan is set by the wake goroutine before the spawn (doWake, |
| @@ -659,14 +670,20 @@ const ( |
| // sidecarStopTimeout serves two related bounds. First, its name: the |
| // sidecar's own stop at sleep time. `sidecar run` has no drain handler, so |
| // the SIGTERM is effectively immediate, and the SIGKILL fallback behind this |
| -// timeout is insurance. Second (design §6.7): how long stop() waits on |
| -// sidecarStartDone for an in-flight sidecar restart to land before giving up |
| -// and proceeding on whatever e.sidecar currently is. The restart goroutine's |
| -// own entry-identity check makes that second wait purely a promptness |
| -// trade-off, never a correctness one. Nothing is orphaned or misplaced |
| -// whether it fires in time or times out. |
| +// timeout is insurance. Second (design §6.7), as sidecarStartJoinWait's |
| +// default: how long a flag-off stop() waits on sidecarStartDone for an |
| +// in-flight sidecar restart to land before carrying on with whatever |
| +// e.sidecar currently is. |
| const sidecarStopTimeout = 5 * time.Second |
| +// sidecarStartJoinWait is how long a flag-off terminal stop waits for an |
| +// in-flight sidecar start before it carries on with the rest of the stop. |
| +// It is a promptness bound only: a start still in flight then keeps the |
| +// stop pending, with the lease, until it returns and has stopped what it |
| +// started, and the host cgroup is proven empty after that (Task 8, F6 |
| +// plan 5). A var so tests can shrink it. |
| +var sidecarStartJoinWait = sidecarStopTimeout |
| + |
| // sidecarSettle is the standalone worker's readiness window — what stands in |
| // for the health probe a child that binds nothing can never answer. A worker |
| // is up once it either reports ready over the agent socket (NoteWorkerReady) |
| @@ -715,6 +732,22 @@ func stopDiscardedSidecar(sc Proc) { |
| }() |
| } |
| +// stopSidecarNow stops a sidecar a flag-off restart brought back for an |
| +// entry that is stopping (Task 8, F6 plan 5): synchronously, bounded by |
| +// sidecarStopTimeout, so the start's done channel closes only after it. |
| +// The stop's host proof follows it, and with a delegated cgroup that proof |
| +// also reaches whatever this stop left. |
| +func stopSidecarNow(host string, sc Proc) { |
| + if sc == nil { |
| + return |
| + } |
| + ctx, cancel := context.WithTimeout(context.Background(), sidecarStopTimeout) |
| + defer cancel() |
| + if err := sc.Stop(ctx); err != nil { |
| + log.Printf("activator: %s: stop the sidecar a restart brought back for a stopping entry: %v", host, err) |
| + } |
| +} |
| + |
| // discardSidecar is how a sidecar start that lost its entry is shed. With |
| // generations off it is today's stopDiscardedSidecar, fire-and-forget. With |
| // them on it is synchronous and bounded, and the caller runs it before it |
| @@ -1553,6 +1586,7 @@ func (a *Service) wake(e *actEntry) { |
| // 10); otherwise it gets the canonical handle and a number of its |
| // own. |
| e.gen = w.gen |
| + e.launchLayout = w.launchLayout |
| if e.gen.N == 0 && !e.worker { |
| ran := e.gen |
| e.gen = canonicalGen(e.route, nextGen(), w.spawnVersion) |
| @@ -3533,9 +3567,18 @@ func (a *Service) stop(e *actEntry) { |
| // stops it. sidecarStartDone is closed by the restart goroutine when it |
| // finishes, however it finishes, so this waits on that signal directly |
| // rather than polling sidecarStarting on a timer. |
| + // |
| + // With generations off the wait is bounded (sidecarStartJoinWait), but |
| + // a start still in flight past it is not left behind (Task 8, F6 plan |
| + // 5): the rest of the stop runs, and then the stop stays pending, with |
| + // the lease, until the start returns and has stopped what it started |
| + // (the restart goroutine stops it before it closes sidecarStartDone). |
| + // Only then is the host cgroup proven empty and the lease released |
| + // (reapUncontainedStop). |
| a.mu.Lock() |
| startDone, hook := e.sidecarStartDone, a.joinWaiting |
| a.mu.Unlock() |
| + var sidecarJoin chan struct{} |
| if startDone != nil { |
| if hook != nil { |
| hook(e.host, "sidecar start") |
| @@ -3543,10 +3586,18 @@ func (a *Service) stop(e *actEntry) { |
| if a.gens { |
| <-startDone // the start, and every process it left, is accounted for |
| } else { |
| + bound := time.NewTimer(sidecarStartJoinWait) |
| select { |
| case <-startDone: |
| - case <-time.After(sidecarStopTimeout): |
| + case <-bound.C: |
| + // Recorded now, so every pending path below (a leader still |
| + // running, an unproven generation) waits for it too. |
| + sidecarJoin = startDone |
| + a.mu.Lock() |
| + e.sidecarJoin = startDone |
| + a.mu.Unlock() |
| } |
| + bound.Stop() |
| } |
| } |
| @@ -3685,6 +3736,27 @@ func (a *Service) stop(e *actEntry) { |
| log.Printf("activator: stop %s: a respawn or the child it shed is not proven finished; keeping the lease until it is", e.host) |
| return |
| } |
| + if !a.gens && sidecarJoin != nil { |
| + // A sidecar start is still in flight past the bounded wait above |
| + // (Task 8, F6 plan 5): the stop stays pending with the lease until |
| + // it returns, and finishPendingStops proves the host cgroup then. |
| + log.Printf("activator: stop %s: a sidecar start is still in flight; keeping the lease until it returns", e.host) |
| + a.mu.Lock() |
| + a.keepPendingLocked(e, outcome) |
| + a.mu.Unlock() |
| + return |
| + } |
| + if !a.gens && !a.stopHostProven(e) { |
| + // Flag off, the leader's exit says nothing of what it forked (Task |
| + // 8, F6): the host cgroup is proven empty first. Unproven, the stop |
| + // stays pending with the lease, as for ErrStillRunning, and the |
| + // sweep retries the proof (reapUncontainedStop). |
| + log.Printf("activator: stop %s: its host cgroup is not proven empty; keeping the lease until it is", e.host) |
| + a.mu.Lock() |
| + a.keepPendingLocked(e, outcome) |
| + a.mu.Unlock() |
| + return |
| + } |
| if a.gens && !a.releasable(e) { // swapsim:M8 |
| // THE retain-until-proven rule (spec §7.4): unproven, the stop stays |
| // pending with the lease held, exactly as for ErrStillRunning, and |
| @@ -3760,20 +3832,15 @@ func (a *Service) finishPendingStops() { |
| } |
| continue |
| } |
| - if e.proc == nil { |
| - // Not reached: only the barrier and releasable leave a stop |
| - // pending without a process, and both need generations, which |
| - // the arm above takes. Kept as the guard this loop always had, |
| - // so a broken invariant holds the lease rather than panicking. |
| - continue |
| - } |
| - if e.proc.Exited() { |
| - e.pendingStop = false |
| - outcome := e.stopOutcome |
| - if outcome == nil { |
| - outcome = &stopObservation{} |
| + if !a.gens && (e.proc == nil || e.proc.Exited()) { |
| + // Flag off, the main process (if any) exited: the stop finishes |
| + // once a sidecar start it left in flight has returned and the |
| + // host cgroup is proven empty (Task 8, F6; reapUncontainedStop). |
| + // A process still alive falls through to the re-stop cadence. |
| + if !e.genReaping && sidecarStartJoined(e.sidecarJoin) { |
| + e.genReaping = true |
| + go a.reapUncontainedStop(e) |
| } |
| - go a.finishPendingStop(e, outcome) |
| continue |
| } |
| // Still running: spec §6.3 says sweepOnce retries proc.Stop on its |
| @@ -3791,6 +3858,21 @@ func (a *Service) finishPendingStops() { |
| } |
| } |
| +// sidecarStartJoined reports whether a sidecar start a flag-off stop left |
| +// in flight (actEntry.sidecarJoin) has returned: none was, or its done |
| +// channel is closed. |
| +func sidecarStartJoined(join chan struct{}) bool { |
| + if join == nil { |
| + return true |
| + } |
| + select { |
| + case <-join: |
| + return true |
| + default: |
| + return false |
| + } |
| +} |
| + |
| // respawnDoneLocked reports whether e has no supervised respawn in flight. |
| // The caller holds a.mu. |
| func respawnDoneLocked(e *actEntry) bool { |
| @@ -4322,8 +4404,11 @@ func (a *Service) sweepOnce() { |
| ctx, cancel := context.WithTimeout(context.Background(), instancectl.DeadlineControl) |
| sc, err := a.startSidecar(ctx, e0, route, gen, false) |
| cancel() |
| - var discard Proc |
| - defer func() { a.discardSidecar(route, discard) }() // after the unlock, before done closes |
| + var discard, stopNow Proc |
| + defer func() { // after the unlock, before done closes |
| + a.discardSidecar(route, discard) |
| + stopSidecarNow(host, stopNow) |
| + }() |
| a.mu.Lock() |
| defer a.mu.Unlock() |
| if a.entries[host] != e0 { |
| @@ -4357,7 +4442,11 @@ func (a *Service) sweepOnce() { |
| e0.sidecar, e0.sidecarGen = sc, gen.N |
| return |
| } |
| - discard = sc |
| + // Flag off, the stop either waited this start out or |
| + // is pending on it (e0.sidecarJoin, Task 8): sc is |
| + // stopped here, synchronously, before done closes, |
| + // so the stop's host proof runs after it. |
| + stopNow = sc |
| return |
| } |
| if a.gens && e0.gen.N != gen.N { |
diff --git a/internal/activator/activator_test.go b/internal/activator/activator_test.go| index 4539f774b..7a0add4d9 100644 |
| --- a/internal/activator/activator_test.go |
| +++ b/internal/activator/activator_test.go |
| @@ -32,6 +32,7 @@ import ( |
| "golang.org/x/sys/unix" |
| + "github.com/carlosframework/platform/internal/cgroup" |
| "github.com/carlosframework/platform/internal/edge" |
| "github.com/carlosframework/platform/internal/provision" |
| "github.com/carlosframework/platform/internal/registry" |
| @@ -371,6 +372,9 @@ type fakeBackend struct { |
| // guessed duration. |
| sidecarStartEntered chan struct{} |
| sidecarStartEnteredOnce sync.Once |
| + // sidecarLabel, when set, labels the sidecars StartSidecar returns from |
| + // then on ("sidecar" otherwise), so the order log can tell them apart. |
| + sidecarLabel string |
| // order interleaves backend calls and proc stops (start:instance, |
| // start:sidecar, nextDue, stop:sidecar, stop:instance) for the ordering |
| // assertions. |
| @@ -520,7 +524,11 @@ func (b *fakeBackend) StartSidecar(context.Context, registry.Route) (Proc, error |
| if b.sidecarErr != nil { |
| return nil, b.sidecarErr |
| } |
| - p := &fakeProc{label: "sidecar", rec: b} |
| + label := "sidecar" |
| + if b.sidecarLabel != "" { |
| + label = b.sidecarLabel |
| + } |
| + p := &fakeProc{label: label, rec: b} |
| b.sidecars = append(b.sidecars, p) |
| return p, nil |
| } |
| @@ -6312,32 +6320,28 @@ func TestDoWakeRefreshesTheRouteFromTheRegistry(t *testing.T) { |
| } |
| } |
| -// A restart whose StartSidecar call is still stalled must never install |
| -// onto a DIFFERENT wake of the same host once it finally returns. The bug |
| -// it regression-tests: a discard check that compares a per-entry |
| -// generation counter is always 1 on an entry's first wake (entries are |
| -// never reused, so nothing ever bumps it a second time), which means it |
| -// can never tell "this entry" apart from "a completely different entry, |
| -// for the same host, after a stop and rewake, that also just had its own |
| -// first wake". A stale restart could install its result straight over a |
| -// fresh wake's own sidecar. The fix (this test's assertion) compares entry |
| -// identity: the *actEntry pointer captured when the restart launched, not |
| -// a counter on it. |
| -// |
| -// This test legitimately costs several seconds of wall time: stop() itself |
| -// waits on the in-flight restart, bounded by sidecarStopTimeout, before it |
| -// can finish taking the old entry down (that wait is a separate, already |
| -// covered property, see TestSidecarRestartDoesNotHoldTheLockAcrossTheStart |
| -// and stop()'s own doc comment). The file has no seam to shrink |
| -// sidecarStopTimeout for tests, so this accepts the wall time rather than |
| -// adding one for a single regression test. |
| -func TestStaleSidecarRestartNeverInstallsOntoAFreshWake(t *testing.T) { |
| +// A restart whose StartSidecar call is still stalled when its host stops |
| +// (Task 8, F6 plan 5, flag off): stop() waits for it only so long |
| +// (sidecarStartJoinWait), and then the stop stays pending, with the lease, |
| +// until the start returns. The sidecar the start brought back is stopped, |
| +// and only then is the host cgroup proven empty and the lease released. A |
| +// stale restart therefore can no longer land on a fresh wake of the host: |
| +// that wake cannot begin until this entry is gone. The restart still |
| +// compares entry identity (the *actEntry captured at launch) rather than a |
| +// counter, and the fresh wake below still gets its own sidecar. |
| +func TestAStopKeepsTheLeaseUntilAnInFlightSidecarStartReturns(t *testing.T) { |
| + prev := sidecarStartJoinWait |
| + sidecarStartJoinWait = 50 * time.Millisecond |
| + t.Cleanup(func() { sidecarStartJoinWait = prev }) |
| a, b, _, _ := testActivator(t, time.Hour) |
| + h := &hostEmptyFake{fakeBackend: b} |
| + a.backend = h |
| b.sidecarStaged = true |
| host := "stale.example.com" |
| arrangeAwake(t, a, testRoute(host)) |
| - b.lastSidecar().exit() // the sweeper now wants a restart |
| + first := b.lastSidecar() |
| + first.exit() // the sweeper now wants a restart |
| gate := make(chan struct{}) |
| entered := make(chan struct{}) |
| @@ -6348,35 +6352,244 @@ func TestStaleSidecarRestartNeverInstallsOntoAFreshWake(t *testing.T) { |
| a.sweepOnce() // launches the restart; it blocks inside StartSidecar on gate |
| <-entered |
| + rep, inst := b.lastRep(), b.lastProc() |
| + rep.setOnClose(func() { |
| + stale := b.lastSidecar() |
| + if stale == first || stale.stopCalls() < 1 { |
| + t.Error("the lease was released before the in-flight start's sidecar was stopped") |
| + } |
| + ev := b.events() |
| + stopped, proven := slices.Index(ev, "stop:sidecar:late"), slices.Index(ev, "emptyhost") |
| + if stopped < 0 || proven < stopped { |
| + t.Errorf("events %v: want the late sidecar stopped, then the host proven empty, before the release", ev) |
| + } |
| + }) |
| if !a.Bounce(host) { |
| t.Fatal("Bounce did not find the awake entry") |
| } |
| - deadline := time.Now().Add(8 * time.Second) |
| - for a.entryCount() != 0 && time.Now().Before(deadline) { |
| - time.Sleep(20 * time.Millisecond) |
| + waitFor(t, "the instance to stop", inst.isStopped) |
| + time.Sleep(4 * sidecarStartJoinWait) // well past stop()'s own wait |
| + for range 3 { |
| + a.sweepOnce() |
| } |
| - if a.entryCount() != 0 { |
| - t.Fatal("Bounce'd entry never finished stopping (the stalled restart wedged the stop past its bound)") |
| + if rep.isClosed() || a.entryCount() != 1 { |
| + t.Fatalf("closed=%v entries=%d while a sidecar start was in flight: the lease went first", rep.isClosed(), a.entryCount()) |
| + } |
| + if n := h.count(); n != 0 { |
| + t.Fatalf("the host cgroup was proven %d times before the in-flight start returned", n) |
| } |
| - // A fresh wake of the same host gets its own sidecar. This is what the |
| - // stale restart, still stalled, must not be allowed to overwrite. |
| b.mu.Lock() |
| b.sidecarGate = nil // the fresh wake's own co-spawn must not block on it too |
| b.mu.Unlock() |
| - arrangeAwake(t, a, testRoute(host)) |
| - freshSidecar := b.lastSidecar() |
| - wantStarts := b.sidecarStartCount() + 1 // + the stale call, about to land |
| - |
| + b.lateSidecarLabel("sidecar:late") |
| close(gate) // let the stale StartSidecar call finally return |
| - waitFor(t, "the stale restart to resolve", func() bool { return b.sidecarStartCount() == wantStarts }) |
| - stale := b.lastSidecar() |
| - waitFor(t, "the stale sidecar to be stopped", func() bool { return stale.stopCalls() >= 1 }) |
| + waitFor(t, "the stop to finish once the start returned", func() bool { |
| + a.sweepOnce() |
| + return rep.isClosed() && a.entryCount() == 0 |
| + }) |
| + b.lateSidecarLabel("") |
| + // A fresh wake of the same host gets its own sidecar. |
| + arrangeAwake(t, a, testRoute(host)) |
| + freshSidecar := b.lastSidecar() |
| a.mu.Lock() |
| installed := a.entries[host].sidecar |
| a.mu.Unlock() |
| - if installed != freshSidecar { |
| - t.Fatal("the stale restart installed over the fresh wake's own sidecar") |
| + if installed != freshSidecar || freshSidecar.label != "sidecar" { |
| + t.Fatal("the fresh wake did not get its own sidecar") |
| + } |
| +} |
| + |
| +// hostEmptyFake is fakeBackend with Task 8's host-cgroup proof |
| +// (hostEmptier), the flag-off terminal stop's: err is its answer, and each |
| +// call is counted and logged as "emptyhost" in the backend's order log. |
| +type hostEmptyFake struct { |
| + *fakeBackend |
| + mu sync.Mutex |
| + err error |
| + calls int |
| + passes int // calls that answered nil |
| +} |
| + |
| +func (h *hostEmptyFake) EmptyHost(context.Context, registry.Route) error { |
| + h.mu.Lock() |
| + h.calls++ |
| + err := h.err |
| + if err == nil { |
| + h.passes++ |
| + } |
| + h.mu.Unlock() |
| + h.record("emptyhost") |
| + return err |
| +} |
| + |
| +func (h *hostEmptyFake) passed() int { |
| + h.mu.Lock() |
| + defer h.mu.Unlock() |
| + return h.passes |
| +} |
| + |
| +func (h *hostEmptyFake) set(err error) { |
| + h.mu.Lock() |
| + h.err = err |
| + h.mu.Unlock() |
| +} |
| + |
| +func (h *hostEmptyFake) count() int { |
| + h.mu.Lock() |
| + defer h.mu.Unlock() |
| + return h.calls |
| +} |
| + |
| +// lateSidecarLabel labels every sidecar StartSidecar returns from now on. |
| +func (b *fakeBackend) lateSidecarLabel(label string) { |
| + b.mu.Lock() |
| + b.sidecarLabel = label |
| + b.mu.Unlock() |
| +} |
| + |
| +// Task 8 (F6, flag off): an idle stop proves the host cgroup empty before |
| +// the lease goes. While the proof fails the stop stays pending with the |
| +// lease held; the sweep retries, and the lease goes only after a proof |
| +// that holds. |
| +func TestAFlagOffStopKeepsTheLeaseUntilTheHostCgroupIsProvenEmpty(t *testing.T) { |
| + a, b, clk, _ := testActivator(t, time.Minute) |
| + h := &hostEmptyFake{fakeBackend: b} |
| + a.backend = h |
| + arrangeAwake(t, a, testRoute("f6.example.com")) |
| + h.set(errors.New("the host cgroup still holds a process")) |
| + |
| + clk.advance(2 * time.Minute) |
| + a.sweepOnce() |
| + waitFor(t, "the stop to go pending on the proof", func() bool { |
| + a.mu.Lock() |
| + defer a.mu.Unlock() |
| + e := a.entries["f6.example.com"] |
| + return h.count() >= 1 && e != nil && e.pendingStop && !e.genReaping |
| + }) |
| + rep := b.lastRep() |
| + if rep.isClosed() || a.entryCount() != 1 { |
| + t.Fatal("the lease was released over a host cgroup not proven empty") |
| + } |
| + |
| + rep.setOnClose(func() { |
| + if n := h.passed(); n != 1 { |
| + t.Errorf("passing proofs before the release = %d, want 1", n) |
| + } |
| + }) |
| + h.set(nil) |
| + waitFor(t, "the lease to go once the proof holds", func() bool { |
| + a.sweepOnce() |
| + return rep.isClosed() && a.entryCount() == 0 |
| + }) |
| +} |
| + |
| +// A passing proof releases the lease in the stop itself, after the proof. |
| +func TestAFlagOffStopProvesTheHostCgroupBeforeTheLeaseGoes(t *testing.T) { |
| + a, b, clk, _ := testActivator(t, time.Minute) |
| + h := &hostEmptyFake{fakeBackend: b} |
| + a.backend = h |
| + arrangeAwake(t, a, testRoute("proven.example.com")) |
| + rep := b.lastRep() |
| + var atClose atomic.Int32 |
| + atClose.Store(-1) |
| + rep.setOnClose(func() { atClose.Store(int32(h.count())) }) |
| + |
| + clk.advance(2 * time.Minute) |
| + a.sweepOnce() |
| + waitFor(t, "the stop", func() bool { return a.entryCount() == 0 }) |
| + if got := atClose.Load(); got != 1 { |
| + t.Fatalf("proofs before the release = %d, want 1", got) |
| + } |
| +} |
| + |
| +// No delegated cgroup (cgroup.ErrNoDelegation): the leader's exit is the |
| +// only proof there is (ledger 127), and the lease goes at once. |
| +func TestAFlagOffStopWithNoDelegationReleasesAtOnce(t *testing.T) { |
| + a, b, clk, _ := testActivator(t, time.Minute) |
| + h := &hostEmptyFake{fakeBackend: b} |
| + h.set(cgroup.ErrNoDelegation) |
| + a.backend = h |
| + arrangeAwake(t, a, testRoute("undelegated.example.com")) |
| + |
| + clk.advance(2 * time.Minute) |
| + a.sweepOnce() |
| + waitFor(t, "the stop, with no sweep behind it", func() bool { return a.entryCount() == 0 }) |
| + if !b.lastRep().isClosed() || h.count() != 1 { |
| + t.Fatalf("closed=%v proofs=%d, want the lease released after one proof", b.lastRep().isClosed(), h.count()) |
| + } |
| +} |
| + |
| +// A nested row's child ran as a unit in its host slice, not in |
| +// carlos-exec@<host>: its stop asks no host proof, decided from the layout |
| +// the wake launched on (actEntry.launchLayout, set at install). |
| +func TestAFlagOffStopOfANestedRowAsksNoHostProof(t *testing.T) { |
| + a, b, clk, _ := testActivator(t, time.Minute) |
| + b.layoutFn = func(registry.Route) (Layout, string) { return LayoutNested, "" } |
| + h := &hostEmptyFake{fakeBackend: b} |
| + h.set(errors.New("must not be asked")) |
| + a.backend = h |
| + arrangeAwake(t, a, testRoute("nested-stop.example.com")) |
| + |
| + clk.advance(2 * time.Minute) |
| + a.sweepOnce() |
| + waitFor(t, "the stop", func() bool { return a.entryCount() == 0 }) |
| + if n := h.count(); n != 0 || !b.lastRep().isClosed() { |
| + t.Fatalf("proofs=%d closed=%v, want no host proof and the lease released", n, b.lastRep().isClosed()) |
| + } |
| +} |
| + |
| +// EmptyHost's edges: no delegation is ErrNoDelegation (the caller's |
| +// leader-only proof), and a unit-backed row has no host cgroup of ours. |
| +func TestEmptyHostDispatch(t *testing.T) { |
| + eb := testExecBackend(t) |
| + rt := registry.Route{Host: "edge.example.com"} |
| + if err := eb.EmptyHost(context.Background(), rt); !errors.Is(err, cgroup.ErrNoDelegation) { |
| + t.Fatalf("EmptyHost with no cgroup manager = %v, want ErrNoDelegation", err) |
| + } |
| + sb := &SplitBackend{Exec: eb, Unit: &UnitBackend{}} |
| + if err := sb.EmptyHost(context.Background(), rt); !errors.Is(err, cgroup.ErrNoDelegation) { |
| + t.Fatalf("SplitBackend.EmptyHost (exec row) = %v, want the exec side's answer", err) |
| + } |
| + rt.Backing = registry.BackingUnit |
| + if err := sb.EmptyHost(context.Background(), rt); err != nil { |
| + t.Fatalf("SplitBackend.EmptyHost (unit row) = %v, want nil", err) |
| + } |
| +} |
| + |
| +// The pending-stop path (a leader that was still running when stop() asked) |
| +// is held to the same proof once the leader exits. |
| +func TestAPendingFlagOffStopProvesTheHostCgroupBeforeTheLeaseGoes(t *testing.T) { |
| + a, b, clk, _ := testActivator(t, time.Minute) |
| + h := &hostEmptyFake{fakeBackend: b} |
| + a.backend = h |
| + b.procStuck = true |
| + arrangeAwake(t, a, testRoute("pending-f6.example.com")) |
| + h.set(errors.New("the host cgroup still holds a process")) |
| + |
| + clk.advance(2 * time.Minute) |
| + a.sweepOnce() |
| + p := b.lastProc() |
| + waitFor(t, "the stuck stop", func() bool { return p.stopCalls() >= 1 }) |
| + p.exit() |
| + waitFor(t, "the pending stop's proof", func() bool { |
| + a.sweepOnce() |
| + return h.count() >= 1 |
| + }) |
| + rep := b.lastRep() |
| + if rep.isClosed() { |
| + t.Fatal("a pending stop released the lease over a host cgroup not proven empty") |
| } |
| + |
| + rep.setOnClose(func() { |
| + if n := h.passed(); n != 1 { |
| + t.Errorf("passing proofs before the release = %d, want 1", n) |
| + } |
| + }) |
| + h.set(nil) |
| + waitFor(t, "the lease to go once the proof holds", func() bool { |
| + a.sweepOnce() |
| + return rep.isClosed() && a.entryCount() == 0 |
| + }) |
| } |
diff --git a/internal/activator/genbackend.go b/internal/activator/genbackend.go| index 01c2fa852..46f1f133c 100644 |
| --- a/internal/activator/genbackend.go |
| +++ b/internal/activator/genbackend.go |
| @@ -525,3 +525,46 @@ func (b *SplitBackend) KillHost(ctx context.Context, rt registry.Route) error { |
| } |
| return nil |
| } |
| + |
| +// EmptyHost proves rt's own host cgroup empty for a flag-off terminal stop |
| +// (Task 8, F6): it reads cgroup.events and kills only when something is |
| +// left, so a clean stop leaves no kill mark and nothing for the next start |
| +// to recreate. A populated cgroup goes through KillAndEmpty, which the |
| +// next start answers by making it fresh (Task 19). An absent cgroup is |
| +// empty; no delegated cgroup at all is cgroup.ErrNoDelegation, which the |
| +// caller treats as the leader-only proof. |
| +func (b *ExecBackend) EmptyHost(ctx context.Context, rt registry.Route) error { |
| + if b.Cgroups == nil { |
| + return cgroup.ErrNoDelegation |
| + } |
| + c, err := b.Cgroups.OpenCleanup(rt.Host) |
| + if errors.Is(err, cgroup.ErrNoCgroup) { |
| + return nil |
| + } |
| + if err != nil { |
| + return err |
| + } |
| + populated, err := c.Populated() |
| + if err == nil && populated { |
| + err = c.KillAndEmpty(ctx) |
| + } |
| + if cerr := c.Close(); err == nil { |
| + err = cerr |
| + } |
| + return err |
| +} |
| + |
| +// EmptyHost dispatches like KillHost: a unit-backed row has no host cgroup |
| +// of ours. An exec side without the read-first proof gets KillHost. |
| +func (b *SplitBackend) EmptyHost(ctx context.Context, rt registry.Route) error { |
| + if rt.Backing == registry.BackingUnit { |
| + return nil // systemd owns the unit tenant's tree |
| + } |
| + if e, ok := b.Exec.(hostEmptier); ok { |
| + return e.EmptyHost(ctx, rt) |
| + } |
| + if k, ok := b.Exec.(hostKiller); ok { |
| + return k.KillHost(ctx, rt) |
| + } |
| + return nil |
| +} |
diff --git a/internal/activator/opset.go b/internal/activator/opset.go| index e482d35d4..08e1ef81d 100644 |
| --- a/internal/activator/opset.go |
| +++ b/internal/activator/opset.go |
| @@ -160,6 +160,14 @@ type hostKiller interface { |
| KillHost(ctx context.Context, rt registry.Route) error |
| } |
| +// hostEmptier is the exec backend's host-cgroup proof for a flag-off |
| +// terminal stop (Task 8, F6): host's own cgroup is read first and killed |
| +// only when something is left, then waited empty. stopHostProven prefers |
| +// it to hostKiller, so a clean stop leaves no kill mark. |
| +type hostEmptier interface { |
| + EmptyHost(ctx context.Context, rt registry.Route) error |
| +} |
| + |
| // stopMember stops m's process, bounded by actStopTimeout: SIGTERM, the |
| // generation's own graceful drain, then SIGKILL (spec §7.5 step 6). It |
| // reports whether termination is proven, which takes two things: the |
| @@ -348,6 +356,80 @@ func (a *Service) uncontainedHostEmpty(rt registry.Route, g Gen, launched Layout |
| return false |
| } |
| +// stopHostProven is the flag-off half of F6 for a terminal stop (Task 8): |
| +// a flat or foreign exec row's carlos-exec@<host> is proven empty before |
| +// the stop releases the lease. The leader's exit says nothing of a process |
| +// it forked, and with the flag off that process sits in the host cgroup |
| +// with no generation child to reach it. hostEmptier when the backend has |
| +// it (read first, kill only when populated), else hostKiller; a backend |
| +// with neither has no host cgroup, and the leader's exit is all there is. |
| +// A claims-on process proved its own stop; a nested or unit-backed row has |
| +// its own proof (hostCgroupIsTheWakes, on the layout the wake launched). |
| +// |
| +// Killing is safe here for uncontainedHostEmpty's reasons: the entry is |
| +// stopping, its sidecar was stopped first, a supervised respawn and a |
| +// sidecar start are joined before this runs, and nothing of another entry |
| +// lives in that cgroup. A box with no delegated cgroup |
| +// (cgroup.ErrNoDelegation) keeps the leader-only proof with a log line |
| +// (ledger 127). Any other failure answers false, and the stop stays |
| +// pending with its lease. Never called with a.mu held. |
| +func (a *Service) stopHostProven(e *actEntry) bool { |
| + a.mu.Lock() |
| + rt, g, p, launched := e.route, e.gen, e.proc, e.launchLayout |
| + a.mu.Unlock() |
| + if _, ok := p.(*claimProcess); ok { |
| + return true |
| + } |
| + if !a.hostCgroupIsTheWakes(rt, g, launched) { |
| + return true |
| + } |
| + ctx, cancel := context.WithTimeout(context.Background(), instancectl.DeadlineControl) |
| + defer cancel() |
| + var err error |
| + if he, ok := a.backend.(hostEmptier); ok { |
| + err = he.EmptyHost(ctx, rt) |
| + } else if hk, ok := a.backend.(hostKiller); ok { |
| + err = hk.KillHost(ctx, rt) |
| + } else { |
| + return true // a backend with no host cgroup: the leader's exit is all there is |
| + } |
| + if err == nil { |
| + return true |
| + } |
| + if errors.Is(err, cgroup.ErrNoDelegation) { |
| + log.Printf("activator: %s: stopped with no delegated cgroup; its leader's exit is the only proof there is", rt.Host) |
| + return true |
| + } |
| + log.Printf("activator: stop %s: its host cgroup is not proven empty: %v", rt.Host, err) |
| + return false |
| +} |
| + |
| +// reapUncontainedStop finishes a flag-off pending stop whose main process |
| +// has exited (finishPendingStops): the host cgroup is proven empty |
| +// (stopHostProven) before finishPendingStop releases the lease. A failed |
| +// proof leaves the stop pending, and the next sweep tries again. The |
| +// caller set e.genReaping; this clears it. |
| +func (a *Service) reapUncontainedStop(e *actEntry) { |
| + if !a.stopHostProven(e) { |
| + a.mu.Lock() |
| + e.genReaping = false |
| + a.mu.Unlock() |
| + return |
| + } |
| + a.mu.Lock() |
| + e.genReaping, e.pendingStop = false, false |
| + outcome, p := e.stopOutcome, e.proc |
| + a.mu.Unlock() |
| + if outcome == nil { |
| + outcome = &stopObservation{} |
| + } |
| + if p == nil { |
| + a.finishStop(e, outcome) |
| + return |
| + } |
| + a.finishPendingStop(e, outcome) |
| +} |
| + |
| // hostCgroupIsTheWakes reports whether a failed wake of rt may have run in |
| // carlos-exec@<host>, so that cgroup must be proven empty: an exec child, |
| // that is, a flat or foreign row. A unit-backed row runs under systemd |