carlos / platform Public

activator: a flag-off wake proves its host cgroup empty before it restores

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

claude 1 week ago 79032948546f9e4e59a4bc28d68ce6ef9799ef12 parent 5e51c00

4 files changed, +163 −2

internal/activator/activator.go +14 −0
internal/activator/activator_test.go +101 −2
internal/activator/logsink.go +8 −0
internal/activator/opset.go +40 −0

diff --git a/internal/activator/activator.go b/internal/activator/activator.go

index 460c2b8a0..4240d6b1e 100644

--- a/internal/activator/activator.go

+++ b/internal/activator/activator.go

@@ -1949,6 +1949,20 @@ func (a *Service) doWake(ctx context.Context, e *actEntry) (*wokenInstance, erro

w.stopOutcome.leaseReleased = releaseLease(ctx, lease) == nil

return w, ErrFenced

}

+ // 1.5. Flag off, the host cgroup is proven empty under the lease before

+ // anything is restored or started (Task 8 fix round 1, finding 4). The

+ // stop that released the lease proved it, but an agent restart forgets

+ // a pending stop and its kill marks, so after the TTL a leftover can

+ // still populate it. Unproven, the wake is refused and the lease is left

+ // to its TTL, as an unproven seed leaves it: the leftover may still

+ // write the database.

+ if !a.gens {

+ if err := a.wakeHostProven(ctx, e, rt); err != nil {

+ log.Printf("activator: %s: %v; leaving the lease to its TTL", host, err)

+ w.stopOutcome.uncertain = true

+ return w, fmt.Errorf("%s: refusing to wake: %w", host, err)

+ }

// 2. Restore the DB from S3 when the local copy is gone — the restore must

// happen *under* the lock (restoring without it could later shadow newer

// S3 state written by the true holder). When it's still on disk it is the

diff --git a/internal/activator/activator_test.go b/internal/activator/activator_test.go

index 5e9880561..9e53db5b1 100644

--- a/internal/activator/activator_test.go

+++ b/internal/activator/activator_test.go

@@ -6417,6 +6417,7 @@ func TestAStopKeepsTheLeaseUntilAnInFlightSidecarStartReturns(t *testing.T) {

host := "stale.example.com"

arrangeAwake(t, a, testRoute(host))

+ h.reset() // the wake's own proof (finding 4) is not the stop's

first := b.lastSidecar()

first.exit() // the sweeper now wants a restart

@@ -6436,8 +6437,8 @@ func TestAStopKeepsTheLeaseUntilAnInFlightSidecarStartReturns(t *testing.T) {

t.Error("the lease was released before the in-flight start's sidecar was stopped")

}

ev := b.events()

- stopped, proven := slices.Index(ev, "stop:sidecar:late"), slices.Index(ev, "emptyhost")

- if stopped < 0 || proven < stopped {

+ stopped := slices.Index(ev, "stop:sidecar:late")

+ if stopped < 0 || !slices.Contains(ev[stopped:], "emptyhost") {

t.Errorf("events %v: want the late sidecar stopped, then the host proven empty, before the release", ev)

}

})

@@ -6519,6 +6520,100 @@ func (h *hostEmptyFake) count() int {

return h.calls

}

+// reset forgets the proofs made so far: a stop test calls it after its wake,

+// whose own flag-off proof (finding 4) it does not count.

+func (h *hostEmptyFake) reset() {

+ h.mu.Lock()

+ h.calls, h.passes = 0, 0

+ h.mu.Unlock()

+// Task 8 fix round 1, finding 4: after an agent restart the in-memory

+// pending stop and kill marks are gone, so a flag-off wake proves the host

+// cgroup empty itself, under its lease and before Restore. A proof that

+// fails refuses the wake before anything is restored or started, and keeps

+// the lease (a leftover may still write the database). One that holds lets

+// the wake go on.

+func TestAFlagOffWakeProvesTheHostCgroupBeforeItRestores(t *testing.T) {

+ a, b, _, _ := testActivator(t, time.Minute)

+ h := &hostEmptyFake{fakeBackend: b}

+ a.backend = h

+ h.set(errors.New("a leftover still populates the host cgroup"))

+ rt := testRoute("leftover.example.com")

+ if _, err := a.beginRequest(context.Background(), rt); !errors.Is(err, errHostUnproven) {

+ t.Fatalf("wake = %v, want errHostUnproven", err)

+ }

+ if _, restores, starts := b.counts(); restores != 0 || starts != 0 {

+ t.Fatalf("restores=%d starts=%d over a host cgroup not proven empty, want neither", restores, starts)

+ }

+ if got := b.releaseCount(); got != 0 {

+ t.Fatalf("lease releases = %d, want 0: a leftover may still write the database", got)

+ }

+ if got := memberSafeWakeReason(fmt.Errorf("leftover.example.com: refusing to wake: %w", errHostUnproven)); got != memberSafeWakeReason(ErrFenced) {

+ t.Fatalf("member reads %q, want the fence's existing text", got)

+ }

+ h.set(nil)

+ if _, err := a.beginRequest(context.Background(), rt); err != nil {

+ t.Fatalf("wake = %v once the proof holds", err)

+ }

+ ev := b.events()

+ last := func(name string) int {

+ for i := len(ev) - 1; i >= 0; i-- {

+ if ev[i] == name {

+ return i

+ }

+ return -1

+ }

+ proven, restored := last("emptyhost"), last("restore")

+ if proven < 0 || restored < proven {

+ t.Fatalf("events %v: want the host proven empty before the restore", ev)

+ }

+// No delegated cgroup: the wake goes on as today. A row whose child is not

+// in carlos-exec@<host> (nested, unit-backed) asks no proof, and neither

+// does a wake with generations on, whose proofs are its own.

+func TestAWakeAsksTheHostProofOnlyFlagOffForAnExecChild(t *testing.T) {

+ t.Run("no delegation", func(t *testing.T) {

+ a, b, _, _ := testActivator(t, time.Minute)

+ h := &hostEmptyFake{fakeBackend: b}

+ h.set(cgroup.ErrNoDelegation)

+ a.backend = h

+ if _, err := a.beginRequest(context.Background(), testRoute("undelegated-wake.example.com")); err != nil {

+ t.Fatalf("wake = %v, want today's wake with no delegation", err)

+ }

+ if h.count() != 1 {

+ t.Fatalf("proofs = %d, want 1", h.count())

+ }

+ })

+ for _, tc := range []struct {

+ name string

+ setup func(*Service, *fakeBackend, *registry.Route)

+ }{

+ {"nested", func(_ *Service, b *fakeBackend, _ *registry.Route) {

+ b.layoutFn = func(registry.Route) (Layout, string) { return LayoutNested, "" }

+ }},

+ {"unit-backed", func(_ *Service, _ *fakeBackend, rt *registry.Route) { rt.Backing = registry.BackingUnit }},

+ {"generations on", func(a *Service, _ *fakeBackend, _ *registry.Route) { a.gens = true }},

+ } {

+ t.Run(tc.name, func(t *testing.T) {

+ a, b, _, _ := testActivator(t, time.Minute)

+ h := &hostEmptyFake{fakeBackend: b}

+ h.set(errors.New("must not be asked"))

+ a.backend = h

+ rt := testRoute("no-host-proof.example.com")

+ tc.setup(a, b, &rt)

+ _, _ = a.beginRequest(context.Background(), rt)

+ if n := h.count(); n != 0 {

+ t.Fatalf("host proofs = %d, want none", n)

+ }

+ })

+ }

// lateSidecarLabel labels every sidecar StartSidecar returns from now on.

func (b *fakeBackend) lateSidecarLabel(label string) {

b.mu.Lock()

@@ -6535,6 +6630,7 @@ func TestAFlagOffStopKeepsTheLeaseUntilTheHostCgroupIsProvenEmpty(t *testing.T)

h := &hostEmptyFake{fakeBackend: b}

a.backend = h

arrangeAwake(t, a, testRoute("f6.example.com"))

+ h.reset() // the wake's own proof (finding 4) is not the stop's

h.set(errors.New("the host cgroup still holds a process"))

clk.advance(2 * time.Minute)

@@ -6568,6 +6664,7 @@ func TestAFlagOffStopProvesTheHostCgroupBeforeTheLeaseGoes(t *testing.T) {

h := &hostEmptyFake{fakeBackend: b}

a.backend = h

arrangeAwake(t, a, testRoute("proven.example.com"))

+ h.reset() // the wake's own proof (finding 4) is not the stop's

rep := b.lastRep()

var atClose atomic.Int32

atClose.Store(-1)

@@ -6589,6 +6686,7 @@ func TestAFlagOffStopWithNoDelegationReleasesAtOnce(t *testing.T) {

h.set(cgroup.ErrNoDelegation)

a.backend = h

arrangeAwake(t, a, testRoute("undelegated.example.com"))

+ h.reset() // the wake's own proof (finding 4) is not the stop's

clk.advance(2 * time.Minute)

a.sweepOnce()

@@ -6663,6 +6761,7 @@ func TestAPendingFlagOffStopProvesTheHostCgroupBeforeTheLeaseGoes(t *testing.T)

a.backend = h

b.procStuck = true

arrangeAwake(t, a, testRoute("pending-f6.example.com"))

+ h.reset() // the wake's own proof (finding 4) is not the stop's

h.set(errors.New("the host cgroup still holds a process"))

clk.advance(2 * time.Minute)

diff --git a/internal/activator/logsink.go b/internal/activator/logsink.go

index 049dd0cd6..114b833ad 100644

--- a/internal/activator/logsink.go

+++ b/internal/activator/logsink.go

@@ -109,6 +109,14 @@ func memberSafeWakeReason(err error) string {

// member gets the maintenance fence's existing text.

err = ErrFenced

}

+ if errors.Is(err, errHostUnproven) {

+ // A flag-off wake refused because its host cgroup holds processes

+ // not proven gone (Task 8 fix round 1, finding 4): the platform holds

+ // the host closed until its own proof holds, and it reopens by

+ // itself. The fence's existing text; the cgroup detail is the

+ // operator's (the wake logs it).

+ err = ErrFenced

+ }

if errors.Is(err, errWakeUnproven) || genStartUnproven(err) || errors.Is(err, errOutcomeUndelivered) {

// A wake refused while the host's last failed wake is not proven

// stopped (zero-downtime spec §7.4), or that failed wake itself: a

diff --git a/internal/activator/opset.go b/internal/activator/opset.go

index 6439d29c9..94e55bf39 100644

--- a/internal/activator/opset.go

+++ b/internal/activator/opset.go

@@ -9,6 +9,7 @@ package activator

import (

"context"

"errors"

+ "fmt"

"log"

"time"

@@ -404,6 +405,41 @@ func (a *Service) stopHostProven(e *actEntry) bool {

return false

}

+// wakeHostProven is a flag-off wake's proof, under its lease and before

+// Restore (Task 8 fix round 1, finding 4), that carlos-exec@<host> holds

+// nothing: EmptyHost reads cgroup.events and kills only what is left, so a

+// clean host costs one read and leaves no kill mark. Only a row whose child

+// runs there is asked: an exec child (flat or foreign) or a worker, which

+// spawns there whatever its layout says; a nested or unit-backed row runs

+// elsewhere and has its own proofs. A backend with no hostEmptier has no

+// host cgroup to read. No delegated cgroup (cgroup.ErrNoDelegation) is

+// today's wake. Any other failure is errHostUnproven.

+//

+// Killing is safe here for uncontainedHostEmpty's reasons: the wake holds

+// the host's lease, nothing of it has run yet, and the stop that released

+// the lease before it proved its own processes gone, so whatever is left

+// belongs to no live entry. Never called with a.mu held.

+func (a *Service) wakeHostProven(ctx context.Context, e *actEntry, rt registry.Route) error {

+ he, ok := a.backend.(hostEmptier)

+ if !ok || rt.Backing == registry.BackingUnit {

+ return nil

+ }

+ if !e.worker {

+ if c, ok := a.backend.(Classifier); ok {

+ if l, _ := c.Layout(rt); l == LayoutNested {

+ return nil

+ }

+ pctx, cancel := context.WithTimeout(context.WithoutCancel(ctx), instancectl.DeadlineControl)

+ defer cancel()

+ err := he.EmptyHost(pctx, rt)

+ if err == nil || errors.Is(err, cgroup.ErrNoDelegation) {

+ return nil

+ }

+ return fmt.Errorf("%w: %v", errHostUnproven, err)

// hostCgroupIsTheWakes reports whether a failed wake of rt may have run in

// carlos-exec@<host>, so that cgroup must be proven empty: an exec child,

// that is, a flat or foreign row. A unit-backed row runs under systemd

@@ -702,6 +738,10 @@ type unprovenWake struct {

// proven stopped.

var errWakeUnproven = errors.New("the last failed wake's generation is not proven stopped")

+// errHostUnproven refuses a flag-off wake whose host cgroup holds processes

+// it could not prove gone (wakeHostProven).

+var errHostUnproven = errors.New("the host cgroup holds processes not proven gone")

// genStartUnproven reports whether a generation start's error says the

// start failed and its kill did not prove the generation gone: plan 2's

// errGenUnproven (startUnitAt) or root's instancectl.ErrGenUnproven. The