amadan

carlos / platform Public

zdd plan 3 task 4: the overlap swap, spawn by staged path, health by handle, switch, retire

Service.Swap and swapOverlap (spec §7.5 steps 1-7): admit and register
the successor under the §7.4 checks, capture each artifact's tier, raise
the parent, re-check at the exec, spawn through spawnSuccessor, health by
the generation's own socket, switch e.gen, AppSocket, the canonical link
and the cache generation under a.mu, then retire the old generation
through the operation set. The one outcome classifier (classifyStep,
verdictOf, claimAdmission, stepResult).

Adapted per the controller: verdictOf also reads the strict-host gate
errors, ErrClaimIsolationUnavailable, errUnitNoGens, errNestedStaged,
errSwapNoGens and errSwapStopped as NoOwnership. spawnSuccessor and the
claimSpawn seam land here; with no seam a claims-on swap is refused as a
claims admission (abandoned, nothing spawned, never the claims-off
StartStaged), since every claims-on launch is closed on this base.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 06de1aaf48a4129ec069acd2564bc80ac1b853d8 parent f51b076
4 files changed, +1164 −1
  • internal/activator/activator.go +9 −0
  • internal/activator/swap.go +431 −1
  • internal/activator/swap_fixture_test.go +265 −0
  • internal/activator/swap_overlap_test.go +459 −0
diff --git a/internal/activator/activator.go b/internal/activator/activator.go
index 59c2775a7..9192a5447 100644
--- a/internal/activator/activator.go
+++ b/internal/activator/activator.go
@@ -773,6 +773,15 @@ type Service struct {
// names them. Both under a.mu.
genPending map[pendingGeneration]pendingBuild
genPendingSeq uint64
+ // genHealth, when set, replaces a generation's health check. It is a
+ // test seam. Nil is waitHealthyCtx on the generation's own socket (spec
+ // §7.1: health uses the handle, never AppSocket).
+ genHealth func(ctx context.Context, g Gen) error
+ // claimSpawn, when set, replaces a claims-on successor's spawn. It is a
+ // test seam: a claims-on spawn needs a delegated cgroup tree and an open
+ // isolated launch. Nil in production, where every claims-on launch is
+ // closed on this base and spawnSuccessor refuses it.
+ claimSpawn func(ctx context.Context, e *actEntry, rt registry.Route, work hostwrite.Work, g Gen) (Gen, Proc, error)
}
// New wires the state machine to its side-effect backend and due-time store.
diff --git a/internal/activator/swap.go b/internal/activator/swap.go
index beb1a58f5..75536c291 100644
--- a/internal/activator/swap.go
+++ b/internal/activator/swap.go
@@ -4,7 +4,19 @@
// generations switch off, it refuses every attempt and spawns nothing.
package activator
-import "time"
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log"
+ "time"
+
+ "github.com/carlosframework/platform/internal/cgroup"
+ "github.com/carlosframework/platform/internal/hostbox"
+ "github.com/carlosframework/platform/internal/hostguard"
+ "github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/registry"
+)
// Cause is why an attempt exists (spec §6.3).
type Cause string
@@ -66,3 +78,421 @@ type SwapResult struct {
// a failed hold is held down (spec §6.4, §8.3). Swap sets it.
Mode SwapMode
}
+
+var (
+ errSwapOff = errors.New("swap: off on this box")
+ errSwapBusy = errors.New("swap: another generation operation is in flight")
+ errSwapKind = errors.New("swap: only exec instances overlap; workers and sidecars keep stop-then-start")
+ errSuccessorExited = errors.New("swap: successor exited before the switch")
+ errSwapMode = errors.New("swap: unknown mode")
+)
+
+func abandon(err error) SwapResult { return SwapResult{Outcome: SwapAbandoned, Reason: err.Error()} }
+func failSwap(err error) SwapResult { return SwapResult{Outcome: SwapFailed, Reason: err.Error()} }
+
+// stepOutcome is how one step of an attempt (a spawn, a health check, a
+// readiness or settle window, a pinned wake) ended.
+type stepOutcome int
+
+const (
+ stepOK stepOutcome = iota
+ stepCancelled // the attempt's lifecycle ended it: its context, the barrier, a fence, an intentional stop
+ stepStale // the attempt gate refused it: a newer attempt, a halted cohort, a lost lease, shutdown
+ stepFailed // the build failed: spawn, health or readiness
+)
+
+// classifyStep is THE outcome classifier (spec §6.4, §7.4): every path
+// that might record a build failure asks it first, the swaps here and plan
+// 5's planned wakes alike. Only stepFailed is a failure of the build; a
+// cancelled or stale step records nothing and falls back to nothing.
+// lifecycle is the attempt's own context (nil when the caller has none). A
+// health deadline that expires is the build's failure (DeadlineExceeded);
+// the lifecycle's own end, or a cancellation, is not.
+func classifyStep(lifecycle context.Context, err error) stepOutcome {
+ switch {
+ case err == nil:
+ return stepOK
+ case lifecycle != nil && lifecycle.Err() != nil,
+ errors.Is(err, context.Canceled), errors.Is(err, errSwapStopped), errors.Is(err, ErrFenced):
+ return stepCancelled
+ case verdictOf(err) != gateProceed:
+ // The attempt gate's own refusal, wherever it surfaced: at Step 3a,
+ // or as a claims validation inside the work at the exec. Nothing of
+ // the build ran because of it (spec §7.4).
+ return stepStale
+ }
+ return stepFailed
+}
+
+// gateVerdict is the attempt gate's answer, typed (spec §6.3, §7.4).
+type gateVerdict int
+
+const (
+ // gateProceed: the attempt is current and owns the host.
+ gateProceed gateVerdict = iota
+ // gateSuperseded: still the owner, but a newer attempt or a halted
+ // cohort replaced this one. A restore of what it replaced may run, and
+ // only behind a fresh ownership check at that exec.
+ gateSuperseded
+ // gateNoOwnership: the entry is not current, the lease is lost, a fence
+ // or the agent's shutdown holds the host, or the claims validation
+ // refused (a changed stamp, a durable fence, a changed claim). Nothing
+ // may spawn, a restore included.
+ gateNoOwnership
+)
+
+// verdictOf types an attempt-gate error. An error that is not the gate's
+// (a failure of the build) is gateProceed: the gate did not refuse.
+//
+// Besides the gate's own refusals, three kinds of refusal that stop a spawn
+// before anything of the build runs are NoOwnership, so the one classifier
+// reads them as abandonment, never as a failed build:
+// - plan 2's strict-host gate on placed spawns (index P2-8):
+// ErrStrictRequired, ErrStrictGroupInUse, ErrStrictCleanupPending;
+// - a closed claims-on launch (index P2-9): ErrClaimIsolationUnavailable.
+// It wraps hostbox.ErrUnsupported, which is not otherwise classified,
+// so it is named here;
+// - a row that cannot run a staged generation at all: errUnitNoGens,
+// errNestedStaged, errSwapNoGens.
+//
+// errSwapStopped (the barrier's cancel, from memberCurrentLocked) is
+// NoOwnership too: the entry is stopping, so nothing may spawn for it.
+// classifyStep reads it as stepCancelled before it asks this.
+func verdictOf(err error) gateVerdict {
+ switch {
+ case err == nil:
+ return gateProceed
+ case errors.Is(err, errSwapSuperseded):
+ return gateSuperseded
+ case errors.Is(err, errSwapNotAwake), errors.Is(err, errSwapLeaseLost), errors.Is(err, errSwapFenced),
+ errors.Is(err, errSwapShutdown), errors.Is(err, errSwapStopped), errors.Is(err, ErrFenced),
+ errors.Is(err, hostguard.ErrStampMismatch), errors.Is(err, hostguard.ErrFenced),
+ errors.Is(err, hostguard.ErrInactive), errors.Is(err, hostguard.ErrRetiredStamp),
+ errors.Is(err, hostguard.ErrOldStamp), errors.Is(err, hostguard.ErrClaimIncarnation),
+ errors.Is(err, hostguard.ErrRecordIncarnation), errors.Is(err, hostguard.ErrNotParticipant),
+ errors.Is(err, hostguard.ErrNoClaim), errors.Is(err, hostguard.ErrNoClaims),
+ errors.Is(err, hostguard.ErrNoHolder), errors.Is(err, hostguard.ErrMalformedClaim),
+ errors.Is(err, hostbox.ErrChanged), errors.Is(err, errClaimsAdmission),
+ errors.Is(err, ErrStrictRequired), errors.Is(err, ErrStrictGroupInUse), errors.Is(err, ErrStrictCleanupPending),
+ errors.Is(err, ErrClaimIsolationUnavailable),
+ errors.Is(err, errUnitNoGens), errors.Is(err, errNestedStaged), errors.Is(err, errSwapNoGens):
+ return gateNoOwnership
+ }
+ return gateProceed
+}
+
+// errClaimsAdmission marks an error a claims-on spawn met before its exec
+// hook ran (claimAdmission): the claim read failed (a store error, a
+// timeout), the claim was absent or malformed, or the work's validation
+// refused before the spawn. Nothing of the build ran, so it is never a
+// failure of the build (spec §7.4).
+var errClaimsAdmission = errors.New("the claims admission refused before the exec")
+
+// claimAdmission wraps err in errClaimsAdmission when the spawn never
+// reached its exec hook (reached false). An error after the hook ran is the
+// spawn's own, and is returned as it is.
+func claimAdmission(err error, reached bool) error {
+ if err == nil || reached {
+ return err
+ }
+ return fmt.Errorf("%w: %w", errClaimsAdmission, err)
+}
+
+// stepResult is classifyStep for a swap step: a cancelled or stale step
+// abandons the attempt (over is true); a failure of the build is the
+// caller's to report (over is false).
+func stepResult(lifecycle context.Context, err error) (SwapResult, bool) {
+ switch classifyStep(lifecycle, err) {
+ case stepCancelled:
+ return abandon(errSwapStopped), true
+ case stepStale:
+ return abandon(err), true
+ }
+ return SwapResult{}, false
+}
+
+// Swap runs one attempt on an awake entry (spec §7.3–§7.5, §7.8). It
+// blocks until the attempt is done, failed or abandoned. For an overlap
+// that includes the retiring generation's drain. For a hold it lasts until
+// the new generation is healthy or has failed.
+//
+// newest reports the scheduler's newest revision for a host. It is called
+// with a.mu held, so it must not block and must not call back into the
+// Service.
+func (a *Service) Swap(ctx context.Context, at Attempt, newest func(host string) uint64) SwapResult {
+ if !a.gens {
+ return abandon(errSwapOff)
+ }
+ gb, ok := a.backend.(SwapBackend)
+ if !ok {
+ return abandon(errSwapNoGens)
+ }
+ if newest == nil {
+ return abandon(errSwapSuperseded)
+ }
+ switch at.Mode {
+ case ModeOverlap:
+ return a.swapOverlap(ctx, gb, at, newest)
+ }
+ return abandon(errSwapMode)
+}
+
+// attemptCtx is the attempt's own lifetime. It ends with the caller's ctx,
+// with the agent (Shutdown cancels lifeCtx), or with the barrier, which
+// calls the returned cancel through the member.
+func (a *Service) attemptCtx(ctx context.Context) (context.Context, context.CancelFunc) {
+ mctx, cancel := context.WithCancel(ctx)
+ if a.lifeCtx == nil {
+ return mctx, cancel
+ }
+ stop := context.AfterFunc(a.lifeCtx, cancel)
+ return mctx, func() { stop(); cancel() }
+}
+
+// genHealthy is the successor's health check, through its own handle:
+// GET / below 500 within spawnHealthTimeout (spec §7.5 step 4).
+func (a *Service) genHealthy(ctx context.Context, g Gen) error {
+ if a.genHealth != nil {
+ return a.genHealth(ctx, g)
+ }
+ if g.Socket == "" {
+ return errors.New("generation has no socket")
+ }
+ return waitHealthyCtx(ctx, g.Socket, spawnHealthTimeout)
+}
+
+// spawnSuccessor is §7.5 step 3 in both modes. Claims off, it is the
+// backend's staged spawn. Claims on, a claim-backed spawn would run inside
+// the selected work, which validates the claim and stamp at the exec.
+//
+// gate is the attempt gate (memberCurrentLocked, taking a.mu itself). It
+// runs immediately before every exec: claims off, right before
+// StartStaged. Claims on, it belongs after the runtime's Read and Prepare
+// and again inside the selected work at the exec, as the spawn's before
+// hook (plan 3 Task 10).
+//
+// Every claims-on launch is closed on this base (index P2-9): the isolated
+// root launcher cannot carry claim reservations. So a claims-on spawn is
+// refused before anything runs, as a claims admission (NoOwnership, the
+// attempt abandoned), and is never sent down the claims-off StartStaged.
+// claimSpawn is the test seam that stands in for the open launch.
+func (a *Service) spawnSuccessor(ctx context.Context, gb SwapBackend, e *actEntry, rt registry.Route, work hostwrite.Work, g Gen, gate func() error) (Gen, Proc, error) {
+ if hostwrite.Legacy(work) {
+ if err := gate(); err != nil {
+ return g, nil, err
+ }
+ return gb.StartStaged(ctx, rt, g)
+ }
+ if a.claimSpawn != nil {
+ return a.claimSpawn(ctx, e, rt, work, g)
+ }
+ return g, nil, claimAdmission(ErrClaimIsolationUnavailable, false)
+}
+
+// swapOverlap is §7.5 steps 1 to 7. The successor is registered in the
+// operation set under the same lock that runs the §7.4 checks, so a
+// terminal stop that begins later always sees it and joins it.
+//
+// Every exit after the admission leaves the set through dropMember (or, on
+// success, promotes the successor and drops the retiring generation), and
+// leaveSet restores a raised parent when the last member leaves. Nothing
+// here blocks under a.mu except GenSwitched, which plan 1's lock contract
+// puts in the same critical section as e.gen.
+func (a *Service) swapOverlap(ctx context.Context, gb SwapBackend, at Attempt, newest func(string) uint64) SwapResult {
+ // The bytes first. An artifact that is not what the attempt chose never
+ // reaches admission, let alone exec (spec §4.2, §7.5 step 3).
+ if err := verifyStaged(at.StagedPath, at.Digest); err != nil {
+ return abandon(fmt.Errorf("staging: %w", err))
+ }
+ work, err := a.selectWork(ctx, at.Host)
+ if err != nil {
+ return abandon(fmt.Errorf("admission: %w", err))
+ }
+ mctx, cancel := a.attemptCtx(ctx)
+ defer cancel()
+ m := &genMember{
+ gen: Gen{N: nextGen(), Build: at.Build, StagedPath: at.StagedPath, Digest: at.Digest, Demand: at.Demand},
+ role: roleSpawning,
+ cancel: cancel,
+ done: make(chan struct{}),
+ }
+ var (
+ e *actEntry
+ rt registry.Route
+ incumbent Gen // the current generation at admission, for its tier
+ )
+ // Step 1: the checks and joining the set, inside the selected work. Only
+ // the registration runs there. A claims-on spawn re-enters the work
+ // itself (Task 10), and hostguard's flock does not nest. Registering
+ // under the same lock as the checks is what lets a stop that begins
+ // afterwards see the successor and join it.
+ err = a.admit(ctx, work, func() error {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ e = a.entries[at.Host]
+ if err := a.swapChecksLocked(e, at, newest); err != nil {
+ return err
+ }
+ if e.worker || e.route.Kind != "instance" {
+ return errSwapKind
+ }
+ // A foreign row keeps its canonical socket (plan 1), so two
+ // generations of it cannot coexist. It may still hold-swap.
+ if c, ok := a.backend.(Classifier); ok {
+ if l, _ := c.Layout(e.route); l != LayoutFlat && l != LayoutNested {
+ return errSwapLayout
+ }
+ }
+ if len(e.ops.members) > 0 {
+ return errSwapBusy
+ }
+ a.addMemberLocked(e, m)
+ rt, incumbent = e.route, e.gen
+ return nil
+ })
+ if err != nil {
+ // The work may refuse after its body registered the member.
+ if a.isMember(e, m) {
+ a.dropMember(e, m)
+ }
+ return abandon(err)
+ }
+
+ // Each artifact's own tier, captured now (spec §7.3): the incumbent's
+ // from the build it runs, the successor's from the attempt's build.
+ // Neither is read from the live link, which root may repoint. They size
+ // the overlap parent, the successor's child, and the restore (leaveSet
+ // restores to the current generation's Tier).
+ if incumbent.Tier == (cgroup.Limit{}) {
+ incumbent.Tier = gb.GenTier(rt, incumbent.Build)
+ }
+ succTier := gb.GenTier(rt, at.Build)
+ a.mu.Lock()
+ if e.gen.N == incumbent.N {
+ e.gen.Tier = incumbent.Tier
+ }
+ m.gen.Tier = succTier
+ a.mu.Unlock()
+
+ // Step 2: the parent raise, before the successor spawns (spec §7.3).
+ if err := gb.RaiseGenParent(ctx, rt, incumbent.Tier, succTier); err != nil {
+ a.dropMember(e, m)
+ return abandon(fmt.Errorf("raise parent limit: %w", err))
+ }
+ a.mu.Lock()
+ e.ops.raised = true
+ a.mu.Unlock()
+
+ // Step 3a: the §7.4 checks again, at the exec (spec §6.3, §7.4). The
+ // raise (and plan 5's pin publication, just above this step) took time:
+ // a newer attempt, a halted cohort, a lost lease or a fence since
+ // admission means nothing execs. Claims on, this runs inside the
+ // selected work as the admission did.
+ gate := func() error {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ return a.memberCurrentLocked(e, m, mctx, at, newest)
+ }
+ if err := a.admit(ctx, work, gate); err != nil {
+ a.dropMember(e, m) // restores the raised parent (leaveSet)
+ return abandon(err)
+ }
+
+ // Step 3: exec the attempt's staged build by path. The backend reports
+ // what it ran, and anything but the attempt's choice is refused.
+ g, proc, spawnErr := a.spawnSuccessor(mctx, gb, e, rt, work, m.gen, gate)
+ if spawnErr == nil && (g.N != m.gen.N || g.Build != m.gen.Build || g.StagedPath != m.gen.StagedPath) {
+ spawnErr = fmt.Errorf("backend ran generation %d of %s from %q; the attempt chose %d of %s from %q",
+ g.N, g.Build, g.StagedPath, m.gen.N, m.gen.Build, m.gen.StagedPath)
+ }
+ a.mu.Lock()
+ m.proc = proc
+ if spawnErr == nil {
+ m.gen, m.role = g, roleSuccessor
+ }
+ a.mu.Unlock()
+ if spawnErr != nil {
+ a.dropMember(e, m)
+ if res, over := stepResult(mctx, spawnErr); over {
+ return res
+ }
+ return failSwap(fmt.Errorf("spawn: %w", spawnErr))
+ }
+
+ // Step 4: the health check, through the generation's own handle.
+ if err := a.genHealthy(mctx, m.gen); err != nil {
+ a.dropMember(e, m)
+ if res, over := stepResult(mctx, err); over {
+ return res
+ }
+ return failSwap(fmt.Errorf("health: %w", err))
+ }
+
+ // Step 5: the switch. The checks run again, then, under the entry lock,
+ // AppSocket, the serving build and the cache generation all move with
+ // e.gen (contract C2).
+ var (
+ retiring *genMember
+ switched bool
+ )
+ err = a.admit(ctx, work, func() error {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ if err := a.memberCurrentLocked(e, m, mctx, at, newest); err != nil {
+ return err
+ }
+ if m.proc == nil || m.proc.Exited() {
+ return errSuccessorExited
+ }
+ // The canonical symlink, the purge and the socket mode first. If the
+ // link cannot move, nothing has switched yet. This runs under a.mu,
+ // in the same critical section that sets e.gen below: plan 1's lock
+ // contract for swapCanonicalLink.
+ if err := gb.GenSwitched(e.route, m.gen, m.proc); err != nil {
+ return err
+ }
+ retiring = &genMember{gen: e.gen, role: roleRetiring, proc: e.proc, done: make(chan struct{})}
+ delete(e.ops.members, m.gen.N)
+ a.addMemberLocked(e, retiring) // swapsim:M2
+ e.gen, e.proc = m.gen, m.proc
+ e.spawnVersion, e.staleSince, e.spawnedAt = m.gen.Build, time.Time{}, a.now()
+ a.noteCurrent(e.host, m.gen.N) // plan 1's cache generation (C4)
+ switched = true
+ return nil
+ })
+ if err != nil && switched {
+ // The work refused after its body had switched. The successor is
+ // current and serving, so it is never dropped: the swap finishes,
+ // and the retiring generation still drains below.
+ log.Printf("activator: %s: the selected work reported %v after the switch to generation %d; the switch stands", at.Host, err, m.gen.N)
+ } else if err != nil {
+ a.dropMember(e, m)
+ if errors.Is(err, errSuccessorExited) {
+ return failSwap(err)
+ }
+ return abandon(err)
+ }
+ close(m.done) // promoted: it is current now, no longer a member
+ // The retiring generation is no longer current. The edge closes its idle
+ // connections now (spec §7.2), outside every activator lock (C4).
+ a.genRetired(at.Host, retiring.gen)
+
+ // Steps 6 and 7. The retiring generation drains under its own SIGTERM
+ // handling, bounded by actStopTimeout. Once it is proven empty,
+ // leaveSet restores the parent limit. The good build (§8.5) is plan 5's,
+ // recorded from this result.
+ a.mu.Lock()
+ rt = e.route
+ a.mu.Unlock()
+ emitEvent(a.Logs, rt, "info", "instance stopping")
+ res := SwapResult{Outcome: SwapDone, Gen: m.gen}
+ // The drain is timed on the wall clock (not a.now, which tests fake), for
+ // the "swapped" event's drain time (spec §9.4).
+ drainStart := time.Now()
+ proven := a.dropMember(e, retiring)
+ res.Drain = time.Since(drainStart)
+ if !proven {
+ res.Reason = "retiring generation not yet proven stopped; the sweep finishes it"
+ }
+ return res
+}
diff --git a/internal/activator/swap_fixture_test.go b/internal/activator/swap_fixture_test.go
new file mode 100644
index 000000000..2fdb1c807
--- /dev/null
+++ b/internal/activator/swap_fixture_test.go
@@ -0,0 +1,265 @@
+package activator
+
+import (
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/cgroup"
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// fakeSwapBackend is fakeBackend plus the swap seam (SwapBackend). Every call is recorded in
+// the shared order log: raise:<incumbent>+<successor>, startGen:<path>,
+// switch:<n>, restore:<tier>,
+// startSidecarGen:<path>. Generation children are fakeProcs labelled
+// gen:<n>.
+type fakeSwapBackend struct {
+ *fakeBackend
+ gmu sync.Mutex
+ genCalls int
+ gens []Gen
+ genProcs []*fakeProc
+ genErr error
+ genGate chan struct{} // when set, StartGen blocks until it closes or ctx ends
+ raises int
+ restores int
+ // genSocketDir, when set, puts generation sockets there (Task 12), and
+ // genServe binds a real server on each one, stopped with the process.
+ genSocketDir string
+ genServe func(g Gen) (stop func())
+ // Task 7: legacy sidecars started from a generation.
+ sidecarGens []Gen
+ sidecarGenGate chan struct{}
+ // sidecarStopGate, when set, is installed on every sidecar started from
+ // then on: its Stop blocks until the gate closes.
+ sidecarStopGate chan struct{}
+ // sidecarDescendant: a process a sidecar forked is still in its cgroup,
+ // so KillSidecar (Task 7) cannot prove it empty.
+ sidecarDescendant bool
+ // tiers is each build's own tier (GenTier); a build not here is the
+ // default tier.
+ tiers map[string]cgroup.Limit
+ // onRaise, when set, runs inside RaiseGenParent: a test's window between
+ // an attempt's admission and its exec.
+ onRaise func()
+ // genStuck makes every generation spawned from then on a stuck
+ // fakeProc: its Stop answers ErrStillRunning until exit().
+ genStuck bool
+}
+
+func (g *fakeSwapBackend) StartStaged(ctx context.Context, _ registry.Route, gen Gen) (Gen, Proc, error) {
+ g.record("startGen:" + gen.StagedPath)
+ g.gmu.Lock()
+ g.genCalls++
+ gate, err := g.genGate, g.genErr
+ g.gmu.Unlock()
+ if gate != nil {
+ select {
+ case <-gate:
+ case <-ctx.Done():
+ return gen, nil, ctx.Err()
+ }
+ }
+ if err != nil {
+ return gen, nil, err
+ }
+ gen.Socket = fmt.Sprintf("/run/carlos/g/test.%d.sock", gen.N)
+ if g.genSocketDir != "" {
+ gen.Socket = filepath.Join(g.genSocketDir, fmt.Sprintf("g%d.sock", gen.N))
+ }
+ gen.Control = gen.Socket + ".control"
+ g.gmu.Lock()
+ stuck := g.genStuck
+ g.gmu.Unlock()
+ p := &fakeProc{label: fmt.Sprintf("gen:%d", gen.N), rec: g.fakeBackend, stuck: stuck}
+ var proc Proc = p
+ if g.genServe != nil {
+ proc = &genServingProc{fakeProc: p, stop: g.genServe(gen)}
+ }
+ g.gmu.Lock()
+ g.gens = append(g.gens, gen)
+ g.genProcs = append(g.genProcs, p)
+ g.gmu.Unlock()
+ return gen, proc, nil
+}
+
+// GenTier answers from tiers (by build), and the default tier otherwise.
+func (g *fakeSwapBackend) GenTier(_ registry.Route, build string) cgroup.Limit {
+ g.gmu.Lock()
+ defer g.gmu.Unlock()
+ if t, ok := g.tiers[build]; ok {
+ return t
+ }
+ return cgroup.TierDefault
+}
+
+func (g *fakeSwapBackend) GenReady(registry.Route, Gen, Proc) {}
+
+func (g *fakeSwapBackend) RaiseGenParent(_ context.Context, _ registry.Route, inc, succ cgroup.Limit) error {
+ g.record(fmt.Sprintf("raise:%d+%d", inc.MemoryMax, succ.MemoryMax))
+ g.gmu.Lock()
+ g.raises++
+ hook := g.onRaise
+ g.gmu.Unlock()
+ if hook != nil {
+ hook()
+ }
+ return nil
+}
+
+func (g *fakeSwapBackend) RestoreGenParent(_ context.Context, _ registry.Route, tier cgroup.Limit) error {
+ g.record(fmt.Sprintf("restore:%d", tier.MemoryMax))
+ g.gmu.Lock()
+ g.restores++
+ g.gmu.Unlock()
+ return nil
+}
+
+func (g *fakeSwapBackend) GenSwitched(_ registry.Route, gen Gen, _ Proc) error {
+ g.record(fmt.Sprintf("switch:%d", gen.N))
+ return nil
+}
+
+func (g *fakeSwapBackend) calls() (gens, raises, restores int) {
+ g.gmu.Lock()
+ defer g.gmu.Unlock()
+ return g.genCalls, g.raises, g.restores
+}
+
+func (g *fakeSwapBackend) lastGen() (Gen, *fakeProc) {
+ g.gmu.Lock()
+ defer g.gmu.Unlock()
+ if len(g.gens) == 0 {
+ return Gen{}, nil
+ }
+ return g.gens[len(g.gens)-1], g.genProcs[len(g.genProcs)-1]
+}
+
+// genServingProc is a generation child with a real socket server behind it.
+// Plan 1 Task 4 already declares a servingProc test type in this package
+// (canonlink_test.go, wrapping a Proc and an *http.Server), so this one has
+// its own name.
+type genServingProc struct {
+ *fakeProc
+ stop func()
+}
+
+func (p *genServingProc) Stop(ctx context.Context) error {
+ p.stop()
+ return p.fakeProc.Stop(ctx)
+}
+
+// swapFixture is an awake host on a swap-enabled Service whose backend runs
+// generations. The incumbent is generation 1<<40 on build v1, a number
+// nextGen will never hand out in a test. Build v2 is staged on disk with
+// its real digest.
+type swapFixture struct {
+ a *Service
+ b *fakeSwapBackend
+ clk *fakeClock
+ reg *registry.Registry
+ host string
+ incumbent Gen
+ staged string
+ digest string
+ rev atomic.Uint64
+}
+
+func newSwapFixture(t *testing.T) *swapFixture {
+ t.Helper()
+ b := &fakeSwapBackend{fakeBackend: &fakeBackend{}}
+ clk := &fakeClock{t: time.Unix(1_700_000_000, 0)}
+ reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ a := New(Config{Bucket: "test-bucket", Idle: time.Hour}, b, reg)
+ a.gens = true // plan 1's switch, as plan 5 sets it from CARLOS_SWAP
+ a.now = clk.now
+ a.genHealth = func(context.Context, Gen) error { return nil }
+ f := &swapFixture{a: a, b: b, clk: clk, reg: reg, host: "swap.example.com"}
+ // The route's socket and database live in the test's own directory,
+ // never the shared /tmp.
+ rt := testRoute(f.host)
+ rdir := t.TempDir()
+ rt.Socket, rt.DBPath = filepath.Join(rdir, "app.sock"), filepath.Join(rdir, "app.db")
+ arrangeAwake(t, a, rt)
+ f.incumbent = Gen{N: 1 << 40, Socket: "/run/carlos/g/incumbent.sock", Control: "/run/carlos/g/incumbent.sock.control",
+ Build: "v1", StagedPath: "/opt/carlos/versions/v1/app"}
+ setCurrentGen(t, a, f.host, f.incumbent)
+ dir := filepath.Join(t.TempDir(), "versions", "v2")
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ f.staged = filepath.Join(dir, "app")
+ if err := os.WriteFile(f.staged, []byte("build v2"), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ sum := sha256.Sum256([]byte("build v2"))
+ f.digest = hex.EncodeToString(sum[:])
+ f.rev.Store(1)
+ return f
+}
+
+func (f *swapFixture) attempt(mode SwapMode) Attempt {
+ return Attempt{Host: f.host, Cause: CauseRelease, Build: "v2", StagedPath: f.staged, Digest: f.digest, Mode: mode, SchedRev: f.rev.Load()}
+}
+
+func (f *swapFixture) newest(string) uint64 { return f.rev.Load() }
+
+func (f *swapFixture) appSocket(t *testing.T) string {
+ t.Helper()
+ sock, ok := f.a.AppSocket(f.host)
+ if !ok {
+ t.Fatalf("AppSocket(%s) reported no current generation", f.host)
+ }
+ return sock
+}
+
+// setCurrentGen stands in for plan 1's wake, which records the generation
+// it spawned in e.gen (contract C2, C3).
+func setCurrentGen(t *testing.T, a *Service, host string, g Gen) {
+ t.Helper()
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ e := a.entries[host]
+ if e == nil || e.state != actAwake {
+ t.Fatalf("%s is not awake", host)
+ }
+ e.gen, e.spawnVersion = g, g.Build
+}
+
+// genStateForTest is the entry's current generation and its operation set
+// by role.
+func (a *Service) genStateForTest(host string) (Gen, map[uint64]genRole) {
+ a.mu.Lock()
+ defer a.mu.Unlock()
+ e := a.entries[host]
+ if e == nil {
+ return Gen{}, nil
+ }
+ out := make(map[uint64]genRole, len(e.ops.members))
+ for n, m := range e.ops.members {
+ out[n] = m.role
+ }
+ return e.gen, out
+}
+
+// indexPrefix is slices.Index for the first event with prefix p.
+func indexPrefix(ev []string, p string) int {
+ for i, e := range ev {
+ if strings.HasPrefix(e, p) {
+ return i
+ }
+ }
+ return -1
+}
diff --git a/internal/activator/swap_overlap_test.go b/internal/activator/swap_overlap_test.go
new file mode 100644
index 000000000..a7ed526f2
--- /dev/null
+++ b/internal/activator/swap_overlap_test.go
@@ -0,0 +1,459 @@
+package activator
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/cgroup"
+ "github.com/carlosframework/platform/internal/hostguard"
+ "github.com/carlosframework/platform/internal/hostwrite"
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+// The happy path, in spec order: raise, exec the staged path, health
+// through the handle, switch, then SIGTERM the retiring generation, then
+// restore the parent. The live link names v3 (root moved it) and makes no
+// difference: the successor execs the attempt's staged v2 (review focus 2).
+// The retiring generation is signalled only after AppSocket has moved
+// (review focus 1).
+func TestSwapOverlapSwitchesAfterHealthAndRetiresTheIncumbent(t *testing.T) {
+ f := newSwapFixture(t)
+ f.b.execVerFn = func() string { return "v3" }
+ incumbent := f.b.lastProc()
+ res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest)
+ if res.Outcome != SwapDone || res.Gen.Build != "v2" || res.Gen.StagedPath != f.staged {
+ t.Fatalf("Swap = %+v", res)
+ }
+ if got := f.appSocket(t); got != res.Gen.Socket {
+ t.Fatalf("AppSocket = %q, want the successor's %q", got, res.Gen.Socket)
+ }
+ ev := f.b.events()
+ order := []int{
+ indexPrefix(ev, "raise"),
+ indexPrefix(ev, "startGen:"+f.staged),
+ indexPrefix(ev, fmt.Sprintf("switch:%d", res.Gen.N)),
+ indexPrefix(ev, "stop:instance"),
+ // The parent restore. A bare "restore" is the wake's data restore
+ // (fakeBackend.Restore), logged before any of this.
+ indexPrefix(ev, "restore:"),
+ }
+ for i := range order {
+ if order[i] < 0 || (i > 0 && order[i] < order[i-1]) {
+ t.Fatalf("event order %v (indexes %v): want raise, startGen, switch, stop:instance, restore", ev, order)
+ }
+ }
+ if !incumbent.isStopped() {
+ t.Fatal("the retiring generation was never stopped")
+ }
+ cur, members := f.a.genStateForTest(f.host)
+ if cur.N != res.Gen.N || len(members) != 0 {
+ t.Fatalf("after the swap: current %d, members %v", cur.N, members)
+ }
+ if _, raises, restores := f.b.calls(); raises != 1 || restores != 1 {
+ t.Fatalf("raises %d restores %d, want 1 and 1", raises, restores)
+ }
+ if got := f.b.startCount(); got != 1 {
+ t.Fatalf("Start (live link) calls = %d, want only the original wake", got)
+ }
+}
+
+// During the overlap the parent holds the incumbent's tier plus the
+// successor's, each captured from its own artifact (spec §7.3), and the
+// restore is to whichever generation is current afterwards. Here the
+// incumbent (v1) declares children (1 GiB) and the successor (v2) does
+// not (512 MiB). Summing one tier twice gives a 1 GiB parent (or 2 GiB),
+// and restoring to the live tier leaves the wrong limit after a failure.
+func TestSwapOverlapUsesEachArtifactsTier(t *testing.T) {
+ const gib, half = int64(1 << 30), int64(1 << 29)
+ t.Run("switched: restored to the successor's tier", func(t *testing.T) {
+ f := newSwapFixture(t)
+ f.b.tiers = map[string]cgroup.Limit{"v1": cgroup.TierChildren, "v2": cgroup.TierDefault}
+ res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest)
+ if res.Outcome != SwapDone || res.Gen.Tier != cgroup.TierDefault {
+ t.Fatalf("Swap = %+v, want done with the successor's own tier", res)
+ }
+ ev := f.b.events()
+ if indexPrefix(ev, fmt.Sprintf("raise:%d+%d", gib, half)) < 0 || indexPrefix(ev, fmt.Sprintf("restore:%d", half)) < 0 {
+ t.Fatalf("events %v: want raise:%d+%d and restore:%d", ev, gib, half, half)
+ }
+ if g, _ := f.b.lastGen(); g.Tier != cgroup.TierDefault {
+ t.Fatalf("the successor spawned at %v, want its own 512 MiB tier", g.Tier)
+ }
+ })
+ t.Run("failed: restored to the incumbent's tier", func(t *testing.T) {
+ f := newSwapFixture(t)
+ f.b.tiers = map[string]cgroup.Limit{"v1": cgroup.TierChildren, "v2": cgroup.TierDefault}
+ f.a.genHealth = func(context.Context, Gen) error { return errors.New("unhealthy: HTTP 503") }
+ if res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest); res.Outcome != SwapFailed {
+ t.Fatalf("Swap = %+v", res)
+ }
+ if ev := f.b.events(); indexPrefix(ev, fmt.Sprintf("restore:%d", gib)) < 0 {
+ t.Fatalf("events %v: want restore:%d (the incumbent is still current)", ev, gib)
+ }
+ })
+}
+
+// The "swapped" event carries the retiring generation's drain time (spec
+// §9.4), so the result measures it: from SIGTERM to the proven exit.
+func TestSwapOverlapReportsTheDrainTime(t *testing.T) {
+ f := newSwapFixture(t)
+ gate := make(chan struct{})
+ f.b.lastProc().setStopGate(gate)
+ time.AfterFunc(80*time.Millisecond, func() { close(gate) })
+ res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest)
+ if res.Outcome != SwapDone {
+ t.Fatalf("Swap = %+v", res)
+ }
+ if res.Drain < 50*time.Millisecond || res.Drain > 5*time.Second {
+ t.Fatalf("Drain = %v, want the retiring generation's stop (about 80ms)", res.Drain)
+ }
+}
+
+// A successor that fails its health check, fails to spawn, or exits before
+// the switch leaves the incumbent current and untouched (spec §8.1).
+func TestSwapOverlapFailureKeepsTheIncumbent(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ arrange func(f *swapFixture)
+ reason string
+ }{
+ {"unhealthy", func(f *swapFixture) {
+ f.a.genHealth = func(context.Context, Gen) error { return errors.New("unhealthy: HTTP 503") }
+ }, "health"},
+ {"spawn fails", func(f *swapFixture) { f.b.genErr = errors.New("exec format error") }, "spawn"},
+ {"exits before the switch", func(f *swapFixture) {
+ f.a.genHealth = func(context.Context, Gen) error {
+ _, p := f.b.lastGen()
+ p.exit()
+ return nil
+ }
+ }, errSuccessorExited.Error()},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ f := newSwapFixture(t)
+ tc.arrange(f)
+ incumbent := f.b.lastProc()
+ res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest)
+ if res.Outcome != SwapFailed || !strings.Contains(res.Reason, tc.reason) {
+ t.Fatalf("Swap = %+v, want failed with %q", res, tc.reason)
+ }
+ if got := f.appSocket(t); got != f.incumbent.Socket {
+ t.Fatalf("AppSocket moved to %q after a failure", got)
+ }
+ if incumbent.isStopped() {
+ t.Fatal("the incumbent was stopped by a failed swap")
+ }
+ if _, p := f.b.lastGen(); p != nil && !p.isStopped() {
+ t.Fatal("the failed successor was left running")
+ }
+ if _, raises, restores := f.b.calls(); raises != restores {
+ t.Fatalf("raises %d restores %d: the parent limit was not restored", raises, restores)
+ }
+ })
+ }
+}
+
+// Every refusal spawns nothing. Swaps off is the flag-off test.
+func TestSwapRefusals(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ arrange func(t *testing.T, f *swapFixture, at *Attempt)
+ want error
+ }{
+ {"swap off", func(_ *testing.T, f *swapFixture, _ *Attempt) { f.a.gens = false }, errSwapOff},
+ {"asleep", func(_ *testing.T, _ *swapFixture, at *Attempt) { at.Host = "asleep.example.com" }, errSwapNotAwake},
+ {"worker", func(_ *testing.T, f *swapFixture, _ *Attempt) {
+ f.a.mu.Lock()
+ f.a.entries[f.host].worker = true
+ f.a.mu.Unlock()
+ }, errSwapKind},
+ {"busy", func(t *testing.T, f *swapFixture, _ *Attempt) {
+ plantMember(t, f.a, f.host, roleSuccessor, &fakeProc{}, 77)
+ }, errSwapBusy},
+ {"unknown mode", func(_ *testing.T, _ *swapFixture, at *Attempt) { at.Mode = "" }, errSwapMode},
+ {"superseded", func(_ *testing.T, f *swapFixture, _ *Attempt) { f.rev.Add(1) }, errSwapSuperseded},
+ {"fenced", func(_ *testing.T, f *swapFixture, _ *Attempt) {
+ f.a.mu.Lock()
+ f.a.fences = map[string]uint64{f.host: 3}
+ f.a.mu.Unlock()
+ }, errSwapFenced},
+ {"lease lost", func(_ *testing.T, f *swapFixture, _ *Attempt) { f.b.lastRep().setErr(errors.New("renewal failed")) }, errSwapLeaseLost},
+ {"foreign layout", func(_ *testing.T, f *swapFixture, _ *Attempt) {
+ f.b.layoutFn = func(registry.Route) (Layout, string) { return LayoutForeign, "hand-set socket" }
+ }, errSwapLayout},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ f := newSwapFixture(t)
+ at := f.attempt(ModeOverlap)
+ tc.arrange(t, f, &at)
+ res := f.a.Swap(context.Background(), at, f.newest)
+ if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, tc.want.Error()) {
+ t.Fatalf("Swap = %+v, want abandoned: %v", res, tc.want)
+ }
+ if gens, raises, _ := f.b.calls(); gens != 0 || raises != 0 {
+ t.Fatalf("a refused attempt spawned %d generation(s), raised %d time(s)", gens, raises)
+ }
+ })
+ }
+ t.Run("digest mismatch", func(t *testing.T) {
+ f := newSwapFixture(t)
+ at := f.attempt(ModeOverlap)
+ at.Digest = strings.Repeat("0", 64)
+ res := f.a.Swap(context.Background(), at, f.newest)
+ if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, "staging") {
+ t.Fatalf("Swap = %+v, want abandoned as a staging error", res)
+ }
+ if gens, _, _ := f.b.calls(); gens != 0 {
+ t.Fatal("a mismatched artifact reached StartGen")
+ }
+ })
+ t.Run("backend without generations", func(t *testing.T) {
+ a, _, _, _ := testActivator(t, time.Hour)
+ a.gens = true
+ if res := a.Swap(context.Background(), Attempt{Host: "x.example.com", Mode: ModeOverlap}, func(string) uint64 { return 0 }); res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, errSwapNoGens.Error()) {
+ t.Fatalf("Swap = %+v", res)
+ }
+ })
+}
+
+// A stopping entry admits no successor (spec §7.4: "the entry is still
+// awake"). This is the witness for mutation M5.
+func TestSwapWitnessNoAdmissionOntoAStoppingEntry(t *testing.T) {
+ f := newSwapFixture(t)
+ gate := make(chan struct{})
+ f.b.lastProc().setStopGate(gate)
+ f.a.Bounce(f.host)
+ waitFor(t, "the entry to stop", func() bool {
+ f.a.mu.Lock()
+ defer f.a.mu.Unlock()
+ e := f.a.entries[f.host]
+ return e != nil && e.state == actStopping
+ })
+ res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest)
+ close(gate)
+ if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, errSwapNotAwake.Error()) {
+ t.Fatalf("Swap = %+v, want abandoned: not awake", res)
+ }
+ if gens, _, _ := f.b.calls(); gens != 0 {
+ t.Fatalf("StartGen ran %d time(s) on a stopping entry", gens)
+ }
+}
+
+// The one outcome classifier (spec §6.4, §7.4): only a failure of the build
+// is stepFailed. A cancelled lifecycle, an intentional stop and a refusal
+// by the attempt gate record nothing; a health deadline that expires is the
+// build's.
+func TestClassifyStep(t *testing.T) {
+ live, ended := context.Background(), func() context.Context {
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ return ctx
+ }()
+ for _, tc := range []struct {
+ name string
+ ctx context.Context
+ err error
+ want stepOutcome
+ }{
+ {"ok", live, nil, stepOK},
+ {"the lifecycle ended", ended, errors.New("dial: connection refused"), stepCancelled},
+ {"cancelled", live, fmt.Errorf("health: %w", context.Canceled), stepCancelled},
+ {"fenced", live, fmt.Errorf("wake: %w", ErrFenced), stepCancelled},
+ {"the barrier", live, errSwapStopped, stepCancelled},
+ {"superseded", live, fmt.Errorf("pin: %w", errSwapSuperseded), stepStale},
+ {"lease lost", live, errSwapLeaseLost, stepStale},
+ // Claims validation at the exec (spawnClaimedGen's Work.With):
+ // nothing ran, so no failure of the build and no halted cohort.
+ {"a changed stamp at the exec", live, fmt.Errorf("with: %w", hostguard.ErrStampMismatch), stepStale},
+ {"a durable fence at the exec", live, fmt.Errorf("with: %w", hostguard.ErrFenced), stepStale},
+ // The strict-host gate refused a placed spawn (plan 2), and every
+ // claims-on launch is closed on this base: nothing of the build ran.
+ {"the strict gate", live, fmt.Errorf("x: %w: %w", ErrStrictRequired, ErrStrictGroupInUse), stepStale},
+ {"a closed claims launch", live, fmt.Errorf("launch: %w", ErrClaimIsolationUnavailable), stepStale},
+ // A row that cannot run a staged generation at all.
+ {"a unit row", live, errUnitNoGens, stepStale},
+ {"a nested row before Task 9", live, errNestedStaged, stepStale},
+ {"a backend without generations", live, errSwapNoGens, stepStale},
+ {"a health deadline", live, fmt.Errorf("health: %w", context.DeadlineExceeded), stepFailed},
+ {"HTTP 500", live, errors.New("GET / answered 500"), stepFailed},
+ } {
+ if got := classifyStep(tc.ctx, tc.err); got != tc.want {
+ t.Errorf("%s: %d, want %d", tc.name, got, tc.want)
+ }
+ }
+}
+
+// The gate's verdict is typed (spec §6.3, §7.4): a supersession may restore
+// behind a fresh ownership check; a lost lease, a fence, shutdown or a
+// claims refusal stops every spawn.
+func TestVerdictOf(t *testing.T) {
+ for _, tc := range []struct {
+ err error
+ want gateVerdict
+ }{
+ {nil, gateProceed},
+ {errors.New("GET / answered 500"), gateProceed},
+ {fmt.Errorf("gate: %w", errSwapSuperseded), gateSuperseded},
+ {errSwapLeaseLost, gateNoOwnership},
+ {errSwapFenced, gateNoOwnership},
+ {errSwapShutdown, gateNoOwnership},
+ {errSwapNotAwake, gateNoOwnership},
+ // The barrier's cancel (memberCurrentLocked): the entry is stopping,
+ // so nothing may spawn for it, a restore included.
+ {errSwapStopped, gateNoOwnership},
+ {fmt.Errorf("with: %w", hostguard.ErrStampMismatch), gateNoOwnership},
+ {fmt.Errorf("with: %w", hostguard.ErrFenced), gateNoOwnership},
+ // P5 review partial new #3: an absent claim, and a claim read that
+ // failed before the exec, are no failure of the build either.
+ {fmt.Errorf("with: %w", hostguard.ErrNoClaim), gateNoOwnership},
+ {claimAdmission(errors.New("s3: GetObject: request timeout"), false), gateNoOwnership},
+ {claimAdmission(errors.New("GET / answered 500"), true), gateProceed},
+ // Plan 2's strict-host gate on placed spawns (index P2-8), as the
+ // exec backend wraps them.
+ {fmt.Errorf("h: %w", ErrStrictRequired), gateNoOwnership},
+ {fmt.Errorf("h: %w: %w", ErrStrictRequired, ErrStrictGroupInUse), gateNoOwnership},
+ {fmt.Errorf("h: %w", ErrStrictGroupInUse), gateNoOwnership},
+ {fmt.Errorf("h: %w", ErrStrictCleanupPending), gateNoOwnership},
+ // Every claims-on launch is closed on this base (index P2-9). It
+ // wraps hostbox.ErrUnsupported, and is still NoOwnership, raw or
+ // as a claims admission.
+ {ErrClaimIsolationUnavailable, gateNoOwnership},
+ {fmt.Errorf("start: %w", ErrClaimIsolationUnavailable), gateNoOwnership},
+ {claimAdmission(ErrClaimIsolationUnavailable, false), gateNoOwnership},
+ // A row that cannot run a staged generation: nothing staged spawns.
+ {errUnitNoGens, gateNoOwnership},
+ {errNestedStaged, gateNoOwnership},
+ {errSwapNoGens, gateNoOwnership},
+ } {
+ if got := verdictOf(tc.err); got != tc.want {
+ t.Errorf("verdictOf(%v) = %d, want %d", tc.err, got, tc.want)
+ }
+ }
+}
+
+// Spec §6.3, §7.4: the checks run again at the exec, not only at the
+// admission. A newer attempt that lands while the parent is being raised
+// (or, with plan 5, while the pins are being published) means nothing
+// execs, and the raise is undone. Checked only at the admission, the
+// superseded build execs and is abandoned only at the switch.
+func TestSwapOverlapReChecksAtTheExec(t *testing.T) {
+ f := newSwapFixture(t)
+ f.b.onRaise = func() { f.rev.Add(1) }
+ res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest)
+ if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, errSwapSuperseded.Error()) {
+ t.Fatalf("Swap = %+v, want abandoned: superseded", res)
+ }
+ if gens, raises, restores := f.b.calls(); gens != 0 || raises != restores {
+ t.Fatalf("spawned %d, raises %d, restores %d: want nothing exec'd and the raise undone", gens, raises, restores)
+ }
+}
+
+// A newer attempt that lands during the health check is seen at the switch.
+// The successor is abandoned and stopped, and the incumbent keeps serving.
+func TestSwapOverlapSupersededAtTheSwitch(t *testing.T) {
+ f := newSwapFixture(t)
+ f.a.genHealth = func(context.Context, Gen) error { f.rev.Add(1); return nil }
+ res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest)
+ if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, errSwapSuperseded.Error()) {
+ t.Fatalf("Swap = %+v", res)
+ }
+ if got := f.appSocket(t); got != f.incumbent.Socket {
+ t.Fatalf("AppSocket = %q, want the incumbent", got)
+ }
+ if _, p := f.b.lastGen(); !p.isStopped() {
+ t.Fatal("the abandoned successor was left running")
+ }
+}
+
+// useClaimSpawn stands the claimSpawn seam in for the kernel: a claims-on
+// spawn needs a delegated cgroup tree and an open isolated launch, and
+// every claims-on launch is closed on this base.
+func useClaimSpawn(f *swapFixture) {
+ f.a.claimSpawn = func(ctx context.Context, _ *actEntry, rt registry.Route, _ hostwrite.Work, g Gen) (Gen, Proc, error) {
+ return f.b.StartStaged(ctx, rt, g)
+ }
+}
+
+// Claims on: a refused selection or a refused check, before the spawn or
+// at the switch, abandons the attempt and is never retried as claims off.
+func TestSwapClaimsOn(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ sel func() hostwrite.Selector
+ wantGens int
+ }{
+ {"selector refuses", func() hostwrite.Selector { return &swapSelector{err: hostwrite.ErrUnavailable} }, 0},
+ {"no selector", func() hostwrite.Selector { return nil }, 0},
+ {"refused before the spawn", func() hostwrite.Selector { return &swapSelector{work: &swapWork{denyAt: 1}} }, 0},
+ {"refused at the exec", func() hostwrite.Selector { return &swapSelector{work: &swapWork{denyAt: 2}} }, 0},
+ {"refused at the switch", func() hostwrite.Selector { return &swapSelector{work: &swapWork{denyAt: 3}} }, 1},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ f := newSwapFixture(t)
+ f.a.cfg.HostWrites = hostwrite.Config{Enabled: true, Selector: tc.sel()}
+ if tc.wantGens > 0 {
+ useClaimSpawn(f)
+ }
+ res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest)
+ if res.Outcome != SwapAbandoned {
+ t.Fatalf("Swap = %+v, want abandoned", res)
+ }
+ gens, raises, restores := f.b.calls()
+ if gens != tc.wantGens || raises != restores {
+ t.Fatalf("gens %d (want %d), raises %d, restores %d", gens, tc.wantGens, raises, restores)
+ }
+ if got := f.appSocket(t); got != f.incumbent.Socket {
+ t.Fatalf("AppSocket = %q, want the incumbent", got)
+ }
+ if _, p := f.b.lastGen(); p != nil && !p.isStopped() {
+ t.Fatal("a refused successor was left running")
+ }
+ })
+ }
+ t.Run("allowed", func(t *testing.T) {
+ f := newSwapFixture(t)
+ w := &swapWork{}
+ f.a.cfg.HostWrites = hostwrite.Config{Enabled: true, Selector: &swapSelector{work: w}}
+ useClaimSpawn(f)
+ if res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest); res.Outcome != SwapDone {
+ t.Fatalf("Swap = %+v", res)
+ }
+ if w.withCalls() != 3 {
+ t.Fatalf("With calls = %d, want 3 (the admission, the exec and the switch)", w.withCalls())
+ }
+ })
+ // Every claims-on launch is closed on this base (index P2-9). A
+ // claims-on swap that passes every check reaches the spawn and is
+ // refused there: abandoned (never failed, which would halt the cohort
+ // for a build that never ran), nothing spawned, and never sent down the
+ // claims-off StartStaged. The raised parent is restored.
+ t.Run("closed launch, no seam", func(t *testing.T) {
+ f := newSwapFixture(t)
+ w := &swapWork{}
+ f.a.cfg.HostWrites = hostwrite.Config{Enabled: true, Selector: &swapSelector{work: w}}
+ res := f.a.Swap(context.Background(), f.attempt(ModeOverlap), f.newest)
+ if res.Outcome != SwapAbandoned || !strings.Contains(res.Reason, ErrClaimIsolationUnavailable.Error()) {
+ t.Fatalf("Swap = %+v, want abandoned: %v", res, ErrClaimIsolationUnavailable)
+ }
+ gens, raises, restores := f.b.calls()
+ if gens != 0 || raises != 1 || restores != 1 {
+ t.Fatalf("StartStaged %d, raises %d, restores %d: want nothing spawned and the raise undone", gens, raises, restores)
+ }
+ if ev := f.b.events(); indexPrefix(ev, "startGen:") >= 0 {
+ t.Fatalf("events %v: a claims-on swap went down the claims-off spawn", ev)
+ }
+ if w.withCalls() != 2 {
+ t.Fatalf("With calls = %d, want 2 (the admission and the exec)", w.withCalls())
+ }
+ if got := f.appSocket(t); got != f.incumbent.Socket {
+ t.Fatalf("AppSocket = %q, want the incumbent", got)
+ }
+ if _, members := f.a.genStateForTest(f.host); len(members) != 0 {
+ t.Fatalf("members %v left in the operation set", members)
+ }
+ })
+}