carlos / platform Public

Sidecar isolation Task 6 (4/6): RoleGone generalises LegacyRunGone; a failed nested worker wake keeps its lease until its unit is proven gone (D19)

RoleGone(ctx, rt, role) is LegacyRunGone's body for any original-template
unit; LegacyRunGone calls it with the run role, so existing callers are
unchanged. The activator's roleProver seam and roleGone replace the run-only
prover; retained unproven wakes carry the role their proof is of. A failed
wake marked legacy-uncertain chooses the worker role for a worker entry (its
only such start is its worker unit's), never the run unit's and never
KillHost's answer about a slice the unit does not run in.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

claude 13 hours ago bee95bc35e8b81e536634ebdedf96197f01c6d61 parent 8af9818

8 files changed, +269 −64

internal/activator/activator.go +14 −3
internal/activator/activator_test.go +19 −3
internal/activator/backend_exec.go +22 −13
internal/activator/backend_unit.go +13 −7
internal/activator/containment.go +23 −17
internal/activator/maintenance.go +20 −11
internal/activator/opset.go +9 −5
internal/activator/worker_nested_test.go +149 −5

diff --git a/internal/activator/activator.go b/internal/activator/activator.go

index 9bbc96be4..63139231f 100644

--- a/internal/activator/activator.go

+++ b/internal/activator/activator.go

@@ -2037,10 +2037,15 @@ type wokenInstance struct {

// kill was not proven (genStartUnproven). cleanupFailedWake keeps the

// lease until the sweep proves it, whatever a.gens says.

genUnproven bool

- // legacyUncertain: the spawn was a start of the host's original run

- // unit that root may still carry out (legacyStartUncertain, #292).

- // cleanupFailedWake keeps the lease until the unit is proven gone.

+ // legacyUncertain: the spawn was a start of one of the host's

+ // original-template units that root may still carry out

+ // (legacyStartUncertain, #292): the run unit, or a nested worker's

+ // worker unit (sidecar isolation D19). cleanupFailedWake keeps the lease

+ // until the unit is proven gone. legacyRole names which unit the proof

+ // is of; a worker's only such start is its worker unit's, so it is the

+ // entry's kind that decides, not the layout at cleanup time.

legacyUncertain bool

+ legacyRole instancectl.Role

// pend is the build a cold wake captured and published as pending at

// step 2b (settleWakeIdentity; ruling R-COLDPUBLISH). wake() drops it in

// the critical section that ends the wake: by then the entry's

@@ -2549,6 +2554,12 @@ func (a *Service) doWake(ctx context.Context, e *actEntry) (*wokenInstance, erro

// job systemd queued, which cancelling the client does not cancel)

// is stopped and proven gone before its lease goes (#292, P1).

w.legacyUncertain = legacyStartMayRun(err)

+ if w.legacyUncertain {

+ w.legacyRole = instancectl.RoleRun

+ if e.worker {

+ w.legacyRole = instancectl.RoleWorker

+ }

switch {

case proc != nil || w.genUnproven || w.legacyUncertain:

a.cleanupFailedWake(ctx, rt, w, lease)

diff --git a/internal/activator/activator_test.go b/internal/activator/activator_test.go

index b8c04587f..54f468b04 100644

--- a/internal/activator/activator_test.go

+++ b/internal/activator/activator_test.go

@@ -34,6 +34,7 @@ import (

"github.com/carlosframework/platform/internal/cgroup"

"github.com/carlosframework/platform/internal/edge"

+ "github.com/carlosframework/platform/internal/instancectl"

"github.com/carlosframework/platform/internal/lease"

"github.com/carlosframework/platform/internal/provision"

"github.com/carlosframework/platform/internal/registry"

@@ -327,11 +328,25 @@ func (l *fakeLease) RetryRelease(context.Context) error {

return nil

}

-// LegacyRunGone is ExecBackend's: the legacy run unit's stop, which

-// replaces a queued start job, and its proof.

-func (b *fakeBackend) LegacyRunGone(ctx context.Context, _ registry.Route) error {

+// LegacyRunGone is RoleGone for the run unit.

+func (b *fakeBackend) LegacyRunGone(ctx context.Context, rt registry.Route) error {

+ return b.RoleGone(ctx, rt, instancectl.RoleRun)

+// roleGoneRoles is every role RoleGone was asked about, in order.

+func (b *fakeBackend) roleGoneRoles() []instancectl.Role {

+ b.mu.Lock()

+ defer b.mu.Unlock()

+ return append([]instancectl.Role(nil), b.roleGones...)

+// RoleGone is ExecBackend's: the role's unit stopped through root, which

+// replaces a queued start job, and its proof. legacyStops counts every

+// call; roleGones records the role each was asked about.

+func (b *fakeBackend) RoleGone(ctx context.Context, _ registry.Route, role instancectl.Role) error {

b.mu.Lock()

b.legacyStops++

+ b.roleGones = append(b.roleGones, role)

block := b.legacyBlock

b.mu.Unlock()

if block {

@@ -390,6 +405,7 @@ type fakeBackend struct {

// answer (an unproven stop) while set.

legacyStops int

legacyStopErr error

+ roleGones []instancectl.Role

// legacyBlock holds every LegacyRunGone until its context ends (a

// stop that never proves); legacyEnded counts those that returned.

legacyBlock bool

diff --git a/internal/activator/backend_exec.go b/internal/activator/backend_exec.go

index 175fb79e7..abcb6ba9b 100644

--- a/internal/activator/backend_exec.go

+++ b/internal/activator/backend_exec.go

@@ -2605,23 +2605,32 @@ func (b *ExecBackend) startUnitAt(ctx context.Context, rt registry.Route, sock s

return up, spawnVer, nil

}

-// LegacyRunGone stops rt's original run unit (carlos-instance@, generation

-// 0) and proves it gone (#292, Astra P1): the failed-wake cleanup's proof

-// before an aborted start's lease goes back. The stop goes through

-// instance-control's /stop, which runs `systemctl stop`. systemd keeps one

-// job per unit and a stop job replaces a queued start job, so a successful

-// stop also means that no start job is still queued, even while the edge

-// restart that queued it is in progress. Then the unit must read inactive

-// (or failed) with its cgroup empty or gone, the facts the exit observer

-// trusts (observe, design §6.5). Bounded by ctx, which the caller gives a

-// budget of its own; anything short of the proof is an error, and the

-// caller keeps the lease.

+// LegacyRunGone is RoleGone for rt's original run unit (carlos-instance@,

+// generation 0; #292, Astra P1).

func (b *ExecBackend) LegacyRunGone(ctx context.Context, rt registry.Route) error {

+ return b.RoleGone(ctx, rt, instancectl.RoleRun)

+// RoleGone stops rt's unit for role (the original template's, generation 0:

+// the run unit, or the worker unit of sidecar isolation D19) and proves it

+// gone: the failed-wake cleanup's proof before an aborted start's lease goes

+// back, and a replacement's proof before a fallback. The stop goes through

+// instance-control's /stop, which fences that (role, host), cancels and

+// joins the start attempts in flight for it, and runs `systemctl stop`.

+// systemd keeps one job per unit and a stop job replaces a queued start job,

+// so a successful stop also means that no start job is still queued, even

+// while the edge restart that queued it is in progress, and no earlier

+// request can still submit one. Then the unit must read inactive (or

+// failed) with its cgroup empty or gone, the facts the exit observer trusts

+// (observe, design §6.5). Bounded by ctx, which the caller gives a budget of

+// its own; anything short of the proof is an error, and the caller keeps

+// the lease.

+func (b *ExecBackend) RoleGone(ctx context.Context, rt registry.Route, role instancectl.Role) error {

if b.Control == nil {

return fmt.Errorf("%s: instance control channel not configured", rt.Host)

}

- unit := unitName(instancectl.RoleRun, rt.Host, 0)

- if err := b.Control.Stop(ctx, rt.Host, instancectl.RoleRun); err != nil {

+ unit := unitName(role, rt.Host, 0)

+ if err := b.Control.Stop(ctx, rt.Host, role); err != nil {

return fmt.Errorf("stop %s: %w", unit, err)

}

facts := b.unitFacts()

diff --git a/internal/activator/backend_unit.go b/internal/activator/backend_unit.go

index 29e89d1e6..2dc80cac4 100644

--- a/internal/activator/backend_unit.go

+++ b/internal/activator/backend_unit.go

@@ -57,6 +57,7 @@ import (

"time"

"github.com/carlosframework/platform/internal/cgroup"

+ "github.com/carlosframework/platform/internal/instancectl"

"github.com/carlosframework/platform/internal/registry"

"github.com/carlosframework/platform/internal/release"

)

@@ -373,19 +374,24 @@ func (b *SplitBackend) KillGen(ctx context.Context, rt registry.Route, n uint64)

return k.KillGen(ctx, rt, n)

}

-// LegacyRunGone is legacyRunProver for the composed backend (#292): an

-// exec row's nested run unit is the exec side's to stop and prove. A

-// unit-backed row never starts a carlos-instance@ unit, so it refuses

-// rather than claiming a proof.

+// LegacyRunGone is RoleGone for the run unit (#292).

func (b *SplitBackend) LegacyRunGone(ctx context.Context, rt registry.Route) error {

+ return b.RoleGone(ctx, rt, instancectl.RoleRun)

+// RoleGone is roleProver for the composed backend (#292, sidecar isolation

+// D19): an exec row's nested unit is the exec side's to stop and prove. A

+// unit-backed row never starts an instance template's unit, so it refuses

+// rather than claiming a proof.

+func (b *SplitBackend) RoleGone(ctx context.Context, rt registry.Route, role instancectl.Role) error {

if rt.Backing == registry.BackingUnit {

- return fmt.Errorf("%s: a unit-backed row has no instance run unit: %w", rt.Host, errNoLegacyProof)

+ return fmt.Errorf("%s: a unit-backed row has no instance %s unit: %w", rt.Host, role, errNoLegacyProof)

}

- p, ok := b.Exec.(legacyRunProver)

+ p, ok := b.Exec.(roleProver)

if !ok {

return fmt.Errorf("%s: %w", rt.Host, errNoLegacyProof)

}

- return p.LegacyRunGone(ctx, rt)

+ return p.RoleGone(ctx, rt, role)

}

// Layout implements Classifier for the composed backend: a unit-backed row

diff --git a/internal/activator/containment.go b/internal/activator/containment.go

index 97e59bd00..fce2195c6 100644

--- a/internal/activator/containment.go

+++ b/internal/activator/containment.go

@@ -78,31 +78,37 @@ func legacyStartMayRun(err error) bool {

return errors.As(err, &l)

}

-// legacyRunProver is the backend half of that proof: stop the host's

-// original run unit through instance-control and prove it gone (stop

-// answered, unit inactive, its cgroup empty). nil is the proof.

-// ExecBackend has it; SplitBackend passes it to its exec side.

-type legacyRunProver interface {

- LegacyRunGone(ctx context.Context, rt registry.Route) error

+// roleProver is the backend half of that proof: stop one of the host's

+// original-template units (the run unit, #292, or the worker unit, sidecar

+// isolation D19) through instance-control and prove it gone (stop answered,

+// unit inactive, its cgroup empty). nil is the proof. ExecBackend has it;

+// SplitBackend passes it to its exec side.

+type roleProver interface {

+ RoleGone(ctx context.Context, rt registry.Route, role instancectl.Role) error

}

-// errNoLegacyProof: the backend cannot stop a legacy run unit, so nothing

-// proves an aborted start gone and its lease is kept.

-var errNoLegacyProof = errors.New("activator: this backend cannot prove a run unit stopped")

+// errNoLegacyProof: the backend cannot stop the unit, so nothing proves an

+// aborted start gone and its lease is kept.

+var errNoLegacyProof = errors.New("activator: this backend cannot prove a unit stopped")

// legacyStopBudget bounds one legacy proof: the stop call and the wait for

// the unit to read stopped, on a context of its own (a cancelled wake's

// context is often why the start was aborted).

const legacyStopBudget = instancectl.DeadlineControl

-// legacyRunGone runs the backend's proof for rt under legacyStopBudget and

-// never past Shutdown's own budget (Astra, rounds 2 and 3): a proof started

-// once Shutdown has begun runs under its context, and every proof is

-// registered so that one already running when Shutdown began is cancelled

-// the moment that budget ends (cancelProofs). A proof cut short is

-// unproven, so the lease is kept.

+// legacyRunGone is roleGone for the host's original run unit (#292).

func (a *Service) legacyRunGone(rt registry.Route) error {

- p, ok := a.backend.(legacyRunProver)

+ return a.roleGone(rt, instancectl.RoleRun)

+// roleGone runs the backend's proof for rt's unit of role under

+// legacyStopBudget and never past Shutdown's own budget (Astra, rounds 2

+// and 3): a proof started once Shutdown has begun runs under its context,

+// and every proof is registered so that one already running when Shutdown

+// began is cancelled the moment that budget ends (cancelProofs). A proof

+// cut short is unproven, so the lease is kept.

+func (a *Service) roleGone(rt registry.Route, role instancectl.Role) error {

+ p, ok := a.backend.(roleProver)

if !ok {

return errNoLegacyProof

}

@@ -124,7 +130,7 @@ func (a *Service) legacyRunGone(rt registry.Route) error {

a.mu.Unlock()

cancel()

}()

- return p.LegacyRunGone(ctx, rt)

+ return p.RoleGone(ctx, rt, role)

}

// cancelProofs cancels every legacy proof still running: Shutdown arms it

diff --git a/internal/activator/maintenance.go b/internal/activator/maintenance.go

index ce8d24ee3..0e14f7c92 100644

--- a/internal/activator/maintenance.go

+++ b/internal/activator/maintenance.go

@@ -9,6 +9,7 @@ import (

"github.com/carlosframework/platform/internal/controlname"

"github.com/carlosframework/platform/internal/edge"

+ "github.com/carlosframework/platform/internal/instancectl"

"github.com/carlosframework/platform/internal/registry"

)

@@ -374,22 +375,30 @@ func (a *Service) cleanupFailedWake(ctx context.Context, rt registry.Route, w *w

err := w.proc.Stop(stopCtx)

w.stopOutcome.mainStop, w.stopOutcome.mainExited = err == nil, w.proc.Exited()

}

- // An aborted or failed start of the host's original run unit that root

- // may still carry out (#292, Astra P1): cancelling the client's /start

- // leaves a start job systemd queued, and released over it, the lease

- // would let that job launch a writer no lease covers. The unit is

- // stopped (a stop job replaces the queued start) and proven gone on a

- // budget of its own first. Unproven, the lease is KEPT: retained below

- // and retried by the reaper, and at worst left to its TTL. A stall is

- // the safe outcome; two writers are not.

+ // An aborted or failed start of one of the host's original-template

+ // units that root may still carry out (#292, Astra P1; the worker unit

+ // too, sidecar isolation D19): cancelling the client's /start leaves a

+ // start job systemd queued, and released over it, the lease would let

+ // that job launch a writer no lease covers. The unit is stopped (the

+ // fenced /stop, which cancels and joins the handler and replaces the

+ // queued start) and proven gone on a budget of its own first. The role

+ // is the launch's (wokenInstance.legacyRole): a nested worker's proof

+ // is its worker unit's, never the run unit's, and never KillHost's

+ // answer about a slice the unit does not run in. Unproven, the lease

+ // is KEPT: retained below and retried by the reaper, and at worst left

+ // to its TTL. A stall is the safe outcome; two writers are not.

legacy := w.proc == nil && w.legacyUncertain && !w.genUnproven

if legacy {

- if err := a.legacyRunGone(rt); err != nil {

- log.Printf("activator: %s: the aborted start's unit is not proven stopped (%v); keeping the lease until it is", rt.Host, err)

+ role := w.legacyRole

+ if role == "" {

+ role = instancectl.RoleRun

+ }

+ if err := a.roleGone(rt, role); err != nil {

+ log.Printf("activator: %s: the aborted start's %s unit is not proven stopped (%v); keeping the lease until it is", rt.Host, role, err)

w.stopOutcome.mainExited = false

w.stopOutcome.uncertain = true

u := &unprovenWake{rt: rt, gen: w.gen, lease: lease, rep: w.rep, build: w.spawnIdentity,

- layout: w.launchLayout, legacy: true}

+ layout: w.launchLayout, legacy: true, role: role}

// This wake's uncertainty is this retained start's, and its

// proof discharges it (reapUnprovenWake; seeded pre-land

// review, P2).

diff --git a/internal/activator/opset.go b/internal/activator/opset.go

index 27c3d9578..262fa2d0d 100644

--- a/internal/activator/opset.go

+++ b/internal/activator/opset.go

@@ -753,9 +753,12 @@ type unprovenWake struct {

// (wokenInstance.launchLayout), which every retry of the proof decides

// by (finding 6).

layout Layout

- // legacy: an aborted start of the host's original run unit whose

- // queued job may still run (#292, Astra P1). Proven by legacyRunGone.

+ // legacy: an aborted start of one of the host's original-template

+ // units whose queued job may still run (#292, Astra P1): the run unit

+ // of an instance, or the worker unit of a nested worker (sidecar

+ // isolation D19). role names which; proven by roleGone for it.

legacy bool

+ role instancectl.Role

}

// errWakeUnproven refuses a wake while the host's last failed wake is not

@@ -846,8 +849,9 @@ func (a *Service) reapUnprovenWakesLocked() {

// retried with the same conditional delete (giveBack, #292).

// A retained legacy start (#292, Astra P1) has no process and no unit

-// generation: it is proven by stopping its original run unit again

-// (legacyRunGone), never by the generation proof.

+// generation: it is proven by stopping its unit again (roleGone for the

+// role the start was of: the run unit, or a nested worker's worker unit,

+// sidecar isolation D19), never by the generation proof.

func (a *Service) reapUnprovenWake(host string, u *unprovenWake) {

if u.proc != nil && !u.proc.Exited() {

ctx, cancel := context.WithTimeout(context.Background(), actStopTimeout)

@@ -859,7 +863,7 @@ func (a *Service) reapUnprovenWake(host string, u *unprovenWake) {

}

proven := false

if u.legacy {

- proven = a.legacyRunGone(u.rt) == nil

+ proven = a.roleGone(u.rt, u.role) == nil

} else {

proven = a.failedWakeStopped(u.rt, u.gen, u.proc, u.layout)

}

diff --git a/internal/activator/worker_nested_test.go b/internal/activator/worker_nested_test.go

index be1694240..24e4a846c 100644

--- a/internal/activator/worker_nested_test.go

+++ b/internal/activator/worker_nested_test.go

@@ -303,13 +303,17 @@ func unitInstance(unit string) (instancectl.Role, string, bool) {

}

func (s *fakeSystemd) run(ctx context.Context, args ...string) (string, error) {

- s.mu.Lock()

- s.calls = append(s.calls, strings.Join(args, " "))

- hold := s.holdStart

- s.mu.Unlock()

if len(args) == 0 {

return "", errors.New("systemctl: no verb")

}

+ s.mu.Lock()

+ hold := s.holdStart

+ if args[0] != "start" {

+ // A start is recorded only once it is submitted (below): a held one

+ // that its context ends never was.

+ s.calls = append(s.calls, strings.Join(args, " "))

+ }

+ s.mu.Unlock()

switch args[0] {

case "set-property", "reset-failed", "list-jobs", "daemon-reload":

if args[0] == "reset-failed" && len(args) > 1 {

@@ -350,6 +354,9 @@ func (s *fakeSystemd) run(ctx context.Context, args ...string) (string, error) {

if ctx.Err() != nil {

return "", ctx.Err()

}

+ s.mu.Lock()

+ s.calls = append(s.calls, strings.Join(args, " "))

+ s.mu.Unlock()

return "", s.start(args[1])

case "stop":

return "", s.stop(args[1])

@@ -580,7 +587,9 @@ func (h *workerHarness) stage(ver, script string) release.StagedIdentity {

}

// workerUnit is the worker unit's name.

-func (h *workerHarness) workerUnit() string { return instancectl.UnitName(instancectl.RoleWorker, h.host) }

+func (h *workerHarness) workerUnit() string {

+ return instancectl.UnitName(instancectl.RoleWorker, h.host)

// instanceDir is the row's instance directory.

func (h *workerHarness) instanceDir() string { return provision.InstanceDir(h.b.DataDir, h.host) }

@@ -783,3 +792,138 @@ func TestStartWorkerStagedFencedAndConflictRefuse(t *testing.T) {

})

}

+// ---- D19, the agent half: an uncertain worker start keeps the lease -----

+// D19 over the real control server: a worker /start whose client gives up

+// while root's handler is held comes back marked uncertain both ways (the

+// job may still run, and the build may have run), not as a refusal. The

+// proof is RoleGone for the worker role: the fenced /stop cancels and joins

+// the held handler, which never submits once the stop has answered, and the

+// unit then reads inactive with its cgroup empty. KillHost is no proof here.

+func TestNestedWorkerStartHeldPastCancellationIsUncertainUntilRoleGone(t *testing.T) {

+ h := newWorkerHarness(t, nil)

+ h.stage("v1", argv0Script("v1", filepath.Join(h.root, "argv0")))

+ hold := make(chan struct{})

+ h.sd.mu.Lock()

+ h.sd.holdStart = hold

+ h.sd.mu.Unlock()

+ ctx, cancel := context.WithTimeout(context.Background(), 400*time.Millisecond)

+ defer cancel()

+ p, _, err := h.b.StartWorkerStaged(ctx, h.rt, nil, nil)

+ if p != nil || err == nil {

+ t.Fatalf("StartWorkerStaged = %v, %v; want no proc and an error", p, err)

+ }

+ if !legacyStartMayRun(err) || !startMayHaveRun(nil, err) || launchNotStarted(err) {

+ t.Fatalf("a start lost in root's handler must read as uncertain both ways: %v", err)

+ }

+ pctx, pcancel := context.WithTimeout(context.Background(), 5*time.Second)

+ defer pcancel()

+ if err := h.b.RoleGone(pctx, h.rt, instancectl.RoleWorker); err != nil {

+ t.Fatalf("RoleGone: %v\ncalls:\n%s", err, h.sd.joined())

+ }

+ close(hold) // the held handler, were it still alive, is free to submit

+ time.Sleep(200 * time.Millisecond)

+ calls := h.sd.joined()

+ if strings.Contains(calls, "start "+h.workerUnit()) {

+ t.Fatalf("a start was submitted after the stop; calls:\n%s", calls)

+ }

+ if !strings.Contains(calls, "stop "+h.workerUnit()) || strings.Contains(calls, "carlos-instance@") {

+ t.Fatalf("RoleGone must stop the worker unit and nothing else; calls:\n%s", calls)

+ }

+// RoleGone fails, and so keeps the lease, while the worker unit runs or its

+// cgroup still holds a process; and it stops the role it is asked about.

+func TestRoleGoneRefusesWhileTheWorkerUnitRuns(t *testing.T) {

+ t.Parallel()

+ for _, tc := range []struct {

+ name string

+ active string

+ populated bool

+ }{

+ {"active", "active", true},

+ {"failed with processes left", "failed", true},

+ } {

+ t.Run(tc.name, func(t *testing.T) {

+ ctl := &fakeControl{}

+ unit := &fakeUnit{}

+ unit.set(tc.active, "inv", true, tc.populated)

+ b, rt, _ := nestedWorkerBackend(t, ctl, unit)

+ ctx, cancel := context.WithTimeout(context.Background(), 200*time.Millisecond)

+ defer cancel()

+ if err := b.RoleGone(ctx, rt, instancectl.RoleWorker); err == nil {

+ t.Fatal("RoleGone proved a unit that still runs")

+ }

+ if calls := ctl.joined(); !strings.Contains(calls, "stop "+rt.Host+" worker") || strings.Contains(calls, " run") {

+ t.Fatalf("calls:\n%s\nwant the worker unit stopped, never the run unit", calls)

+ }

+ })

+ }

+ t.Run("LegacyRunGone is RoleGone for the run unit", func(t *testing.T) {

+ ctl := &fakeControl{}

+ unit := &fakeUnit{}

+ unit.set("inactive", "", false, false)

+ b, rt := nestedBackend(t, ctl, unit)

+ if err := b.LegacyRunGone(context.Background(), rt); err != nil {

+ t.Fatal(err)

+ }

+ if calls := ctl.joined(); !strings.Contains(calls, "stop "+rt.Host+" run") {

+ t.Fatalf("calls:\n%s", calls)

+ }

+ })

+// D19: a failed nested worker wake whose start is uncertain keeps its lease

+// until RoleGone for the worker role proves the unit gone; the proof is

+// never asked of the run unit, and KillHost's answer is not it. Released

+// only after the proof holds.

+func TestFailedNestedWorkerWakeKeepsLeaseUntilRoleGone(t *testing.T) {

+ fb := &fakeBackend{}

+ b := &stagedFake{fakeBackend: fb}

+ a, _, _, _ := testActivator(t, time.Hour)

+ a.backend = b

+ // The nested arm's answer to a start lost in root's handler.

+ fb.workerErr = markLegacyStartUncertain(markStartUncertain(errors.New("start carlos-instance-worker@bot.service: context deadline exceeded")))

+ fb.legacyStopErr = errors.New("unit is deactivating")

+ rt := testWorkerRoute("bot.x.internal")

+ rt.DBPath = provision.NestedDBPath("/data", rt.Host)

+ if err := a.WakeWorker(rt); err == nil {

+ t.Fatal("WakeWorker succeeded over a lost start")

+ }

+ if rel := fb.releaseCount(); rel != 0 {

+ t.Fatalf("lease releases = %d after an uncertain worker start; the lease must be kept until the unit is proven gone", rel)

+ }

+ if roles := fb.roleGoneRoles(); !slices.Equal(roles, []instancectl.Role{instancectl.RoleWorker}) {

+ t.Fatalf("RoleGone asked for %v, want exactly the worker role once", roles)

+ }

+ a.mu.Lock()

+ u := a.unprovenWakes[rt.Host]

+ a.mu.Unlock()

+ if u == nil || !u.legacy || u.role != instancectl.RoleWorker {

+ t.Fatalf("retained wake %+v, want a legacy-marked worker proof obligation", u)

+ }

+ // Still unproven: the sweep retries, and the lease stays.

+ a.sweepOnce()

+ waitFor(t, "a retried proof", func() bool { return len(fb.roleGoneRoles()) >= 2 })

+ if fb.releaseCount() != 0 {

+ t.Fatal("the lease went back while the proof still failed")

+ }

+ // The proof holds: the lease goes back, and only now.

+ fb.mu.Lock()

+ fb.legacyStopErr = nil

+ fb.mu.Unlock()

+ a.sweepOnce()

+ waitFor(t, "the lease to go back", func() bool { return fb.releaseCount() == 1 })

+ for _, r := range fb.roleGoneRoles() {

+ if r != instancectl.RoleWorker {

+ t.Fatalf("RoleGone asked for %s; a worker's proof is its own unit's", r)

+ }

+ a.mu.Lock()

+ _, retained := a.unprovenWakes[rt.Host]

+ a.mu.Unlock()

+ if retained {

+ t.Fatal("the proven wake is still retained")

+ }