amadan

carlos / platform Public

zdd-5 task 14: wire CARLOS_SWAP=overlap into the agent

swapArming refuses the flag without a delegated cgroup subtree or a
generation-capable control channel (PF15): generations, containment and the
scheduler stay off, every leg stays on today's paths, and the status carries
the reason code. With generations on, armRollout arms the scheduler, or
refuses every managed start (refuseManaged) when the deploy store, release
key or failure book is unavailable, standing the bounce legs down. armAlways
publishes a worker's serving version through a cheap ordered writer.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude aa8295c3ca5a4cd914315e9285c866707a5a5631 parent 18d5623
5 files changed, +1013 −8
  • internal/host/agent.go +41 −2
  • internal/host/restart.go +16 −3
  • internal/host/rollout.go +439 −0
  • internal/host/rollout_test.go +510 −0
  • internal/status/snapshot.go +7 −3
diff --git a/internal/host/agent.go b/internal/host/agent.go
index e325b04f6..4fddde110 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -390,6 +390,16 @@ func runAgent(args []string, buildClaims ClaimFactory) {
log.Printf("agent: %v (nested wakes and the layout pass refuse until the unit is fixed)", err)
}
eb.NestedOK = nestedGate(selfCheck, instancefs.MountNoSymfollow, eb.Control.Ready)
+ // CARLOS_SWAP=overlap turns on generations (plan 1), and with them
+ // plan 3's Swap, before the Service exists, which reads the switch
+ // once in New. Plan 2's per-generation containment goes on with
+ // them: an overlap needs each generation in its own child. A box
+ // that cannot contain generations refuses the flag here and stays
+ // on today's paths (swapArming), so all three stay off.
+ swapOn := swapArming(os.Getenv, eb)
+ activator.SetGenerations(swapOn)
+ eb.GenContainment = swapOn
+ rolloutMaterialize = eb.Materialize
svc = activator.New(*cfg, &activator.SplitBackend{
Exec: eb,
Unit: &activator.UnitBackend{OptDir: release.OptDir()},
@@ -420,6 +430,12 @@ func runAgent(args []string, buildClaims ClaimFactory) {
if logSink != nil {
svc.Logs = logSink
}
+ armAlways(svc, reg, dd) // flag on or off
+ var rolloutSink activator.EventSink
+ if logSink != nil { // a typed nil in the interface would not be nil
+ rolloutSink = logSink
+ }
+ armRollout(agentCtx, os.Getenv, svc, reg, dd, rolloutSink, openDeployStore)
// The residence-move machinery (issue #331), armed with the
// activator: the staging-fallback restore needs a same-region
// mirror client, and the move worker needs the service's quiesce
@@ -1109,6 +1125,12 @@ func syncExecSerialized(ctx context.Context, reg *registry.Registry, store relea
for _, cfg := range writes {
opts = append(opts, appconfig.WithHostWrite(cfg))
}
+ // CARLOS_SWAP=overlap: the scheduler restarts, paced, and acknowledges a
+ // revision once it is active; refusing: no restart. configSyncMu above
+ // is what makes ConfigDelivered serial per host.
+ if cb, ok := rolloutConfigCallbacks(ctx, store); ok {
+ return appconfig.SyncExecWithCallbacks(ctx, reg, store, priv, dd, os.Getenv("PLATFORM_APPS_DOMAIN"), cb, opts...)
+ }
return appconfig.SyncExec(ctx, reg, store, priv, dd, bounce(svc), os.Getenv("PLATFORM_APPS_DOMAIN"), configAckFunc(ctx, store, "agent"), opts...)
}
@@ -2259,7 +2281,9 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service, sd *schedu
}
d.restart = restart
d.passOpts = opts
- if leg := convergeLegFor(svc, reg); leg != nil {
+ // The scheduler's pass with CARLOS_SWAP=overlap armed, nothing while
+ // managed starts are refused, today's convergeAdopted otherwise.
+ if leg := fastConvergeLeg(svc, reg); leg != nil {
go runConvergeLoop(leg, adoptedConvergeEvery, nil)
}
// The serial-triggered full pass set, in spec order: mint routes
@@ -3203,7 +3227,13 @@ func statusLoop(reg *registry.Registry, svc *activator.Service) {
// otherwise.
delay := time.Minute
for {
- time.Sleep(delay)
+ select {
+ case <-time.After(delay):
+ case <-statusNudge:
+ // A rollout transition (flag on only): publish within seconds,
+ // at most every two, so a busy rollout cannot hammer the bucket.
+ time.Sleep(2 * time.Second)
+ }
delay = status.SnapshotPeriod + rand.N(time.Minute)
// Routes are listed here (not left to statusTick's own reg.List())
@@ -3294,6 +3324,9 @@ func statusLoop(reg *registry.Registry, svc *activator.Service) {
// Builds the root pass rolled back (D1), as the metrics loop's
// refusal watch last read them. Private Metrics only: hosts.
p.ReleaseRefusals = releaseRefusalProbe()
+ // The rollout scheduler's report (zero-downtime spec §8.4): nil
+ // with CARLOS_SWAP off. Private Metrics only.
+ p.Rollout = rolloutStatusProbe()
// Hosts held for a dev proof (D3): private Metrics only.
p.ProofWaits = proofWaitProbe()
@@ -3725,6 +3758,12 @@ func adoptBounceFn(svc *activator.Service) func(host string) bool {
if svc == nil {
return nil
}
+ if m := armedRollout(); m != nil {
+ return m.Adopted
+ }
+ if rolloutUnavailable() != "" {
+ return nil // no adopt bounce while managed starts are refused
+ }
return svc.BounceAdopted
}
diff --git a/internal/host/restart.go b/internal/host/restart.go
index e811109b0..57fc71f84 100644
--- a/internal/host/restart.go
+++ b/internal/host/restart.go
@@ -874,8 +874,19 @@ func restartRequestLeg(ctx context.Context, routes []registry.Route, store relea
if len(moved) == 0 && len(scoped) == 0 {
return false
}
- bounceExecRoutes(routes, moved, bounce)
- cycleWorkers(routes, moved, bounce, wake)
+ // CARLOS_SWAP=overlap: each exec restart is the scheduler's attempt,
+ // paced and identified by its request (spec §10). While managed starts
+ // are refused (refuseManaged), no exec restart is made: the wake after
+ // the stop would be refused, and the box's status names why. A box that
+ // declined the flag (swapArming) bounces as today.
+ m := armedRollout()
+ legacy := m == nil && rolloutUnavailable() == ""
+ if m != nil {
+ feedRestarts(routes, moved, scoped, mem.gen, m.Restart)
+ } else if legacy {
+ bounceExecRoutes(routes, moved, bounce)
+ cycleWorkers(routes, moved, bounce, wake)
+ }
// The narrowed half reaches no sidecar at all: scopedHosts gates on
// release.Adoptable, which is Kind=="instance" and stays that way (design
// §9.3 — widening it centrally would enrol sidecars in adopt bookkeeping
@@ -886,7 +897,9 @@ func restartRequestLeg(ctx context.Context, routes []registry.Route, store relea
// design made: narrowed restarts landed on main after this branch was
// written. Reaching a sidecar by name wants its own selector, and is a
// follow-up rather than something to widen here by accident.
- bounceScopedRoutes(routes, scoped, bounce)
+ if legacy {
+ bounceScopedRoutes(routes, scoped, bounce)
+ }
if err := requestRootRestart(dd); err != nil {
log.Printf("agent: request root restart pass: %v", err)
return false
diff --git a/internal/host/rollout.go b/internal/host/rollout.go
new file mode 100644
index 000000000..09eaa4017
--- /dev/null
+++ b/internal/host/rollout.go
@@ -0,0 +1,439 @@
+package host
+
+import (
+ "context"
+ "log"
+ "os"
+ "path/filepath"
+ "runtime"
+ "sync"
+ "sync/atomic"
+ "time"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/appconfig"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/rollout"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+// The reason codes RolloutReport.Unavailable carries (zero-downtime deploys
+// spec §15, §8.4). They are codes, not member-facing text: plan 6 words
+// them.
+const (
+ // The flag is on but the box cannot contain generations, so the
+ // scheduler is not armed and every leg stays on today's paths (swapArming).
+ rolloutNoDelegation = "no-cgroup-delegation"
+ rolloutNoGenControl = "no-generation-control"
+ rolloutGenerationsOff = "generations-off"
+ // The flag is on and generations are on, but the scheduler's required
+ // state is unavailable, so every managed start is refused
+ // (refuseManaged).
+ rolloutNoStore = "no-deploy-store"
+ rolloutNoReleaseKey = "no-release-key"
+ rolloutBookUnreadable = "failure-book-unreadable"
+)
+
+// refuseTexts is the wake refusal each refuseManaged code carries. It reaches
+// a member through the activator's "wake failed:" event, so it names the
+// missing fact and never the raw error, which goes to the agent's log.
+var refuseTexts = map[string]string{
+ rolloutNoStore: "overlap swaps are on but the deploy store is unavailable",
+ rolloutNoReleaseKey: "overlap swaps are on but the release key is unavailable",
+ rolloutBookUnreadable: "overlap swaps are on but the failure book is unreadable",
+}
+
+var rolloutMgr atomic.Pointer[rollout.Manager]
+
+// rolloutMaterialize is the exec backend's Materialize hook, handed to the
+// scheduler so a cold managed host is installed before its wake is planned
+// (Task 10). runAgent sets it before armRollout; nil refuses such a wake.
+var rolloutMaterialize func(ctx context.Context, host string) error
+
+// armedRollout is the box's rollout manager, nil unless CARLOS_SWAP=overlap
+// armed one. Every leg that would bounce an exec child asks it first.
+func armedRollout() *rollout.Manager { return rolloutMgr.Load() }
+
+// statusNudge asks the status loop for an early tick: a rollout transition
+// should reach status/boxes/<label>.json in seconds, not at the next
+// five-minute tick. Never sent to with the flag off. Capacity one, and every
+// send is non-blocking, so a burst of transitions is one nudge.
+var statusNudge = make(chan struct{}, 1)
+
+// nudgeStatus is the scheduler's OnChange: a non-blocking send that
+// coalesces, as Deps.OnChange requires. It never waits for the status loop.
+func nudgeStatus() {
+ select {
+ case statusNudge <- struct{}{}:
+ default:
+ }
+}
+
+// rolloutDown is why CARLOS_SWAP=overlap is set but the scheduler could not
+// arm while generations are on: managed starts are refused and the bounce
+// legs stand down (refuseManaged). Nil when it armed, when the flag is off,
+// or when the box declined the flag outright (rolloutDeclined).
+var rolloutDown atomic.Pointer[string]
+
+// rolloutDeclined is why CARLOS_SWAP=overlap is set but the box stayed on
+// today's paths: it cannot contain generations (swapArming). Unlike
+// rolloutDown nothing is refused and no leg changes; the reason is only
+// published in the box's status.
+var rolloutDeclined atomic.Pointer[string]
+
+// rolloutUnavailable is rolloutDown's reason, "" when there is none. A leg
+// that bounces exec children stands down while it is set.
+func rolloutUnavailable() string {
+ if p := rolloutDown.Load(); p != nil {
+ return *p
+ }
+ return ""
+}
+
+// rolloutUnarmed is rolloutDeclined's reason, "" when there is none.
+func rolloutUnarmed() string {
+ if p := rolloutDeclined.Load(); p != nil {
+ return *p
+ }
+ return ""
+}
+
+func declineRollout(code string) {
+ c := code
+ rolloutDeclined.Store(&c)
+}
+
+// swapDelegation is what this box lacks to contain generations, "" when
+// nothing. A flat or foreign row's generation children live under the
+// agent's delegated cgroup subtree (eb.Cgroups): without it a spawn still
+// runs, unplaced, but KillGen and the parent raise answer
+// cgroup.ErrNoDelegation, so an overlap would be abandoned and a failed first
+// wake's kill would be unproven. A nested row's generations are root's
+// carlos-instance-gen@ units, reached through the instance control channel,
+// which must speak the generation half (activator.GenControl); whether root
+// can run them is asked per wake by the backend's nested gate (eb.NestedOK)
+// and per call by the channel itself, and a failed StopGen is an unproven
+// kill the activator already keeps the lease for. So the box-wide question
+// for nested rows is only whether the channel has the generation verbs.
+func swapDelegation(eb *activator.ExecBackend) string {
+ if eb == nil || eb.Cgroups == nil {
+ return rolloutNoDelegation
+ }
+ if _, ok := eb.Control.(activator.GenControl); !ok {
+ return rolloutNoGenControl
+ }
+ return ""
+}
+
+// swapArming decides, before the activator exists, whether CARLOS_SWAP=overlap
+// turns generations on (spec §15; plan 3's review: missing delegation must
+// fail closed, CODEX-FINDINGS-ZDD-IMPL-3.md:138). With the flag off it is
+// false and says nothing. With the flag on but no usable containment it is
+// false too: generations, generation containment and the scheduler all stay
+// off, every leg keeps today's paths, and the refusal is logged loudly and
+// published as a reason code in the box's status (rolloutStatusProbe).
+func swapArming(getenv func(string) string, eb *activator.ExecBackend) bool {
+ if !rollout.Enabled(getenv) {
+ return false
+ }
+ if why := swapDelegation(eb); why != "" {
+ log.Printf("agent: REFUSING %s=overlap (%s): generation containment needs the agent's delegated cgroup subtree "+
+ "and the control channel's generation units, so this box stays on today's stop-then-wake paths", rollout.SwapEnv, why)
+ declineRollout(why)
+ return false
+ }
+ return true
+}
+
+// refuseManaged is the flag on with the scheduler's required state
+// unavailable (spec §6.4, §8.3, §8.4). The legacy paths are NOT a fallback:
+// a wake from the live link could exec a build the unreadable failure book
+// holds down, including a hold build after its migration failed. So every
+// managed wake is refused before the lease, and an unmanaged one (a named
+// unit, a unit-backed row) wakes as today. The bounce-based legs stand down
+// too (fastConvergeLoop, adoptBounceFn, restartRequestLeg,
+// syncExecSerialized): a stop whose next wake is refused would only take an
+// instance down. So do the activator's own two moves onto the live link: the
+// sweep's convergence stop of an awake instance (OnDrift answers it with
+// nothing, as the armed scheduler answers an unmanaged host) and a worker's
+// recycle converging onto the live link (CoalesceRecycle present and false:
+// the recycle respawns the worker's captured build). The reason code is
+// published in the box's status (rolloutStatusProbe) until the agent
+// restarts with the state readable.
+func refuseManaged(svc *activator.Service, reason string) {
+ log.Printf("agent: %s=overlap but %s: refusing every managed start, and every stop and move onto a new build, until the agent restarts with it readable",
+ rollout.SwapEnv, reason)
+ r := reason
+ rolloutDown.Store(&r)
+ text := refuseTexts[reason]
+ if text == "" {
+ text = "overlap swaps are on but this box cannot run them"
+ }
+ svc.Planner = func(_ context.Context, rt registry.Route) activator.WakePlan {
+ if !rollout.Managed(rt) {
+ return activator.WakePlan{}
+ }
+ return activator.WakePlan{Refuse: text}
+ }
+ svc.OnDrift = func(string) {}
+ svc.CoalesceRecycle = func(string) bool { return false }
+}
+
+// servingWriter is OnWorkerServing's registry write, off the publisher's
+// path. The activator delivers each worker's serving identity under its
+// box-wide servingMu, inside the respawn and before the respawn is marked
+// done, so a slow registry write there would delay every host's publication,
+// a stop and a lease release behind it. publish only records the newest
+// version per host and returns; one drain goroutine at a time writes them.
+// Deliveries arrive in incarnation order (servingMu), each overwrites the
+// host's pending value, and the drain writes a host's values one after
+// another, so the last write for a host is always the newest identity it
+// was handed: an older incarnation's version can never land last. A
+// coalesced intermediate version is never written, which only skips a value
+// that was already stale.
+type servingWriter struct {
+ set func(host, ver string) error
+ mu sync.Mutex
+ pending map[string]string
+ running bool
+}
+
+func newServingWriter(set func(host, ver string) error) *servingWriter {
+ return &servingWriter{set: set, pending: map[string]string{}}
+}
+
+func (w *servingWriter) publish(host, ver string) {
+ w.mu.Lock()
+ w.pending[host] = ver
+ start := !w.running
+ w.running = true
+ w.mu.Unlock()
+ if start {
+ go w.drain()
+ }
+}
+
+func (w *servingWriter) drain() {
+ for {
+ w.mu.Lock()
+ host, ver, found := "", "", false
+ for h, v := range w.pending {
+ host, ver, found = h, v, true
+ break
+ }
+ if !found {
+ w.running = false
+ w.mu.Unlock()
+ return
+ }
+ delete(w.pending, host)
+ w.mu.Unlock()
+ if err := w.set(host, ver); err != nil {
+ log.Printf("agent: %s: record the worker's serving build %q: %v", host, ver, err)
+ }
+ }
+}
+
+// armAlways wires what the activator needs whatever CARLOS_SWAP says: a
+// worker's serving identity (an empty version included) becomes the
+// registry's ServingVersion, which `carlos sidecars ls` reads (spec §7.9,
+// Task 11). The write goes through a servingWriter so the callback is cheap.
+// The generations file is not set here: runAgent sets it flag-independently
+// (plan 4 R17).
+func armAlways(svc *activator.Service, reg *registry.Registry, dd string) {
+ w := newServingWriter(reg.SetServingVersion)
+ svc.OnWorkerServing = func(host string, id release.StagedIdentity) { w.publish(host, id.Version) }
+}
+
+// rolloutPolicy is the scheduler's policy loader (spec §4.1): the app's
+// default config bundle. appconfig.Get answers an error for every store or
+// decode failure, and only a confirmed missing object (os.ErrNotExist) as an
+// empty manifest, which rollout.PolicyFromManifest reads exactly as nil:
+// the defaults. A failed read leaves the policy cache with its last value,
+// or hold for an app it has never read, so a bucket failure can never
+// become an overlap.
+func rolloutPolicy(store release.Store) func(ctx context.Context, acct, app string) (*appconfig.Manifest, error) {
+ return func(ctx context.Context, acct, app string) (*appconfig.Manifest, error) {
+ return appconfig.Get(ctx, store, acct, app)
+ }
+}
+
+// touchStageMarker writes the stage-request marker, as stageRequester does.
+// A failed write costs latency; carlos-adopt.timer is the floor. Task 15's
+// stage relay uses it too.
+func touchStageMarker(dd string) {
+ if err := os.WriteFile(filepath.Join(dd, stageRequestMarker), []byte(time.Now().UTC().Format(time.RFC3339)+"\n"), 0o644); err != nil {
+ log.Printf("agent: stage relay: %v", err)
+ }
+}
+
+// armRollout builds the scheduler when CARLOS_SWAP=overlap (spec §15) and
+// wires it into svc, which must not have started yet. With the flag off it
+// returns nil and touches nothing. With the flag on but svc built without
+// generations (swapArming declined the flag) it returns nil and touches
+// nothing either: a manager is never armed over a Service that cannot run
+// its attempts. With the flag on but no deploy store, no public key or an
+// unreadable failure book, it returns nil and refuses every managed start
+// (refuseManaged): each of those is a fact the scheduler cannot do without
+// (the canonical demand, the retry signal, the records), and today's paths
+// would ignore the failure records.
+func armRollout(ctx context.Context, getenv func(string) string, svc *activator.Service, reg *registry.Registry, dd string,
+ sink activator.EventSink, openStore func(context.Context) (release.Store, error)) *rollout.Manager {
+ if svc == nil || !rollout.Enabled(getenv) {
+ return nil
+ }
+ if !svc.GenerationsOn() {
+ if rolloutUnarmed() == "" {
+ log.Printf("agent: REFUSING %s=overlap: the activator was built without generations", rollout.SwapEnv)
+ declineRollout(rolloutGenerationsOff)
+ }
+ return nil
+ }
+ store, err := openStore(ctx)
+ if err != nil {
+ log.Printf("agent: %s=overlap: deploy store: %v", rollout.SwapEnv, err)
+ refuseManaged(svc, rolloutNoStore)
+ return nil
+ }
+ pub, err := release.ParsePublicKey(getenv("CARLOS_RELEASE_PUBKEY"), "CARLOS_RELEASE_PUBKEY")
+ if err != nil {
+ log.Printf("agent: %s=overlap: release key: %v", rollout.SwapEnv, err)
+ refuseManaged(svc, rolloutNoReleaseKey)
+ return nil
+ }
+ book, err := rollout.OpenFailureBook(dd)
+ if err != nil {
+ log.Printf("agent: %s=overlap: failure book: %v", rollout.SwapEnv, err)
+ refuseManaged(svc, rolloutBookUnreadable)
+ return nil
+ }
+ platform := runtime.GOOS + "-" + runtime.GOARCH
+ m := rollout.NewManager(ctx, rollout.Deps{
+ Act: svc, Routes: reg, Book: book, OptDir: release.OptDir(), DataDir: dd, Box: BoxLabel(),
+ Demand: func(ctx context.Context, acct, app, ver, path string) (string, string, error) {
+ return release.CanonicalDemand(ctx, store, acct, app, ver, platform, path)
+ },
+ Policy: rolloutPolicy(store),
+ // The raw signed read (PF4): only its Version and PromotedAt are
+ // used, to clear failure records.
+ Pointer: func(ctx context.Context, acct, app, channel string) (*release.ChannelPointer, error) {
+ return release.Channel(ctx, store, pub, acct, app, channel)
+ },
+ Events: sink,
+ Materialize: rolloutMaterialize, // the exec backend's hook, set by runAgent
+ Touch: func() { touchStageMarker(dd) },
+ OnChange: nudgeStatus,
+ })
+ svc.Planner = m.Plan
+ svc.OnGen = m.OnGen
+ svc.OnDrift = m.Nudge
+ svc.CoalesceRecycle = m.CoalesceRecycle
+ if routes, err := reg.List(); err == nil {
+ m.Bootstrap(routes) // spec §8.5: awake, healthy instances record their serving build
+ m.Pass(routes)
+ }
+ rolloutMgr.Store(m)
+ log.Printf("agent: overlap swaps armed (%s=overlap)", rollout.SwapEnv)
+ return m
+}
+
+// rolloutLegFor is the converge leg with the flag on (spec §10): the agent
+// pass of spec §6.1, every adoptedConvergeEvery, in place of convergeAdopted.
+func rolloutLegFor(m *rollout.Manager, reg *registry.Registry) func() {
+ return func() {
+ routes, err := reg.List()
+ if err != nil {
+ return
+ }
+ m.Pass(routes)
+ }
+}
+
+// fastConvergeLeg is the leg fastConvergeLoop runs every
+// adoptedConvergeEvery: the scheduler's pass when it is armed, none while
+// managed starts are refused (legacy convergence would stop instances whose
+// next wake is refused), and today's convergeAdopted otherwise, the flag off
+// or declined included.
+func fastConvergeLeg(svc *activator.Service, reg *registry.Registry) func() {
+ if m := armedRollout(); m != nil {
+ return rolloutLegFor(m, reg)
+ }
+ if rolloutUnavailable() != "" {
+ return nil
+ }
+ return convergeLegFor(svc, reg)
+}
+
+// rolloutConfigCallbacks is config sync's callbacks when the scheduler has
+// a say, and false when it has none (the flag off or declined), in which case
+// syncExecSerialized runs today's SyncExec unchanged. Armed: the scheduler
+// restarts exec instances, paced, and a revision is acknowledged only once
+// it is active (spec §8.1, §10); appconfig calls Delivered after the file is
+// written (syncExecHostClaimed). Refusing: delivered and acknowledged as
+// today, but no restart, because the wake after it would be refused.
+func rolloutConfigCallbacks(ctx context.Context, store release.Store) (appconfig.Callbacks, bool) {
+ if m := armedRollout(); m != nil {
+ return appconfig.Callbacks{
+ Restart: func(string) error { return nil },
+ Delivered: m.ConfigDelivered,
+ Ack: m.DeferAck(configAckFunc(ctx, store, "agent")),
+ }, true
+ }
+ if rolloutUnavailable() != "" {
+ return appconfig.Callbacks{Restart: func(string) error { return nil }, Ack: configAckFunc(ctx, store, "agent")}, true
+ }
+ return appconfig.Callbacks{}, false
+}
+
+// feedRestarts is the restart leg's exec half with the flag on (spec §10):
+// each moved app's exec instances and sidecars, and each narrowed request's
+// hosts, get a restart attempt identified by the request (the app-wide
+// generation's etag, or the newest narrowed request's id). The root half,
+// named units and unit backing, is unchanged.
+func feedRestarts(routes []registry.Route, moved [][2]string, scoped []scopedRestart, gen map[[2]string]string, feed func(host, id string) bool) int {
+ want := map[[2]string]bool{}
+ for _, k := range moved {
+ want[k] = true
+ }
+ n := 0
+ for _, rt := range routes {
+ k := [2]string{release.RouteAccount(rt), rt.App}
+ if !want[k] || !restartEligible(rt) || !execBounceable(rt) {
+ continue
+ }
+ if feed(rt.Host, "gen:"+gen[k]) {
+ n++
+ }
+ }
+ for _, w := range scoped {
+ if len(w.requests) == 0 {
+ continue
+ }
+ id := "scope:" + w.requests[len(w.requests)-1].ID
+ for _, host := range scopedHosts(routes, w.app, w.requests, execBounceable) {
+ if feed(host, id) {
+ n++
+ }
+ }
+ }
+ return n
+}
+
+// rolloutStatusProbe feeds status.Probes.Rollout: nil with the flag off, the
+// scheduler's report when it is armed, and only the reason code when the
+// flag is on but the scheduler is not armed (refusing, or declined).
+func rolloutStatusProbe() *status.RolloutReport {
+ m := armedRollout()
+ if m == nil {
+ if why := rolloutUnavailable(); why != "" {
+ return &status.RolloutReport{Unavailable: why}
+ }
+ if why := rolloutUnarmed(); why != "" {
+ return &status.RolloutReport{Unavailable: why}
+ }
+ return nil
+ }
+ r := m.Status()
+ return &r
+}
diff --git a/internal/host/rollout_test.go b/internal/host/rollout_test.go
new file mode 100644
index 000000000..d5ff0daef
--- /dev/null
+++ b/internal/host/rollout_test.go
@@ -0,0 +1,510 @@
+package host
+
+import (
+ "context"
+ "errors"
+ "os"
+ "path/filepath"
+ "reflect"
+ "slices"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/appconfig"
+ "github.com/carlosframework/platform/internal/cgroup"
+ "github.com/carlosframework/platform/internal/instancectl"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/rollout"
+)
+
+// rolloutFixture is an activator over a fresh registry, built with the
+// generations switch as gens says (runAgent sets it from swapArming before
+// New). The package's rollout state is reset when the test ends.
+func rolloutFixture(t *testing.T, gens bool) (*activator.Service, *registry.Registry, string) {
+ t.Helper()
+ prev := activator.GenerationsEnabled()
+ activator.SetGenerations(gens)
+ t.Cleanup(func() { activator.SetGenerations(prev) })
+ resetRollout(t)
+ dd := t.TempDir()
+ reg, err := registry.Open(filepath.Join(dd, "registry.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { reg.Close() })
+ svc := activator.New(activator.Config{Bucket: "b", Idle: time.Hour}, &activator.SplitBackend{}, reg)
+ return svc, reg, dd
+}
+
+func resetRollout(t *testing.T) {
+ t.Helper()
+ clear := func() {
+ rolloutMgr.Store(nil)
+ rolloutDown.Store(nil)
+ rolloutDeclined.Store(nil)
+ select {
+ case <-statusNudge:
+ default:
+ }
+ }
+ clear()
+ t.Cleanup(clear)
+}
+
+func overlapEnv(pub string) func(string) string {
+ env := map[string]string{"CARLOS_SWAP": "overlap", "CARLOS_RELEASE_PUBKEY": pub}
+ return func(k string) string { return env[k] }
+}
+
+func noEnv(string) string { return "" }
+
+func unusedStore(context.Context) (release.Store, error) { return nil, errors.New("unused") }
+
+func funcPtr(f any) uintptr { return reflect.ValueOf(f).Pointer() }
+
+// legsAreTodays asserts every leg the scheduler could take over is today's:
+// the converge leg, the adopt bounce, config sync's callbacks and the
+// restart leg's bounce.
+func legsAreTodays(t *testing.T, svc *activator.Service, reg *registry.Registry) {
+ t.Helper()
+ if fastConvergeLeg(svc, reg) == nil {
+ t.Error("no converge leg: convergeAdopted must run")
+ }
+ if b := adoptBounceFn(svc); b == nil || funcPtr(b) != funcPtr(svc.BounceAdopted) {
+ t.Error("the adopt bounce is not BounceAdopted")
+ }
+ if _, ok := rolloutConfigCallbacks(context.Background(), release.NewDirStore(t.TempDir())); ok {
+ t.Error("config sync left today's SyncExec")
+ }
+ ctx, s := genKit(t)
+ routes := []registry.Route{execRoute("exec.example.com", "bab", "eleven")}
+ var bounced []string
+ bounce := func(h string) bool { bounced = append(bounced, h); return true }
+ mem := newRestartMemory()
+ askRestart(t, ctx, s, "bab", "eleven")
+ restartRequestLeg(ctx, routes, s, t.TempDir(), bounce, nil, mem)
+ askRestart(t, ctx, s, "bab", "eleven")
+ restartRequestLeg(ctx, routes, s, t.TempDir(), bounce, nil, mem)
+ if !slices.Equal(bounced, []string{"exec.example.com"}) {
+ t.Errorf("restart bounced %v, want today's bounce of the exec instance", bounced)
+ }
+}
+
+// Cross-plan review focus 1: the flag off changes nothing.
+func TestTheFlagOffArmsNothing(t *testing.T) {
+ svc, reg, dd := rolloutFixture(t, false)
+ opened := false
+ m := armRollout(context.Background(), noEnv, svc, reg, dd, nil,
+ func(context.Context) (release.Store, error) { opened = true; return nil, errors.New("unused") })
+ if m != nil || armedRollout() != nil || opened {
+ t.Fatal("CARLOS_SWAP unset armed the scheduler")
+ }
+ if svc.Planner != nil || svc.OnGen != nil || svc.OnDrift != nil || svc.CoalesceRecycle != nil || svc.GenerationsFile != "" {
+ t.Fatal("CARLOS_SWAP unset set an activator hook")
+ }
+ if adoptBounceFn(svc) == nil {
+ t.Fatal("the adopt bounce must stay BounceAdopted")
+ }
+ if swapArming(noEnv, nil) || activator.GenerationsEnabled() {
+ t.Fatal("CARLOS_SWAP unset turned generations on")
+ }
+ if r := rolloutStatusProbe(); r != nil {
+ t.Fatalf("status with the flag off = %+v, want nil", r)
+ }
+}
+
+// Carried obligation 5: with the flag off every leg is today's, and the only
+// hook set is armAlways's OnWorkerServing.
+func TestTheFlagOffLegsAreTodays(t *testing.T) {
+ svc, reg, dd := rolloutFixture(t, false)
+ armAlways(svc, reg, dd)
+ armRollout(context.Background(), noEnv, svc, reg, dd, nil, unusedStore)
+ legsAreTodays(t, svc, reg)
+ if len(statusNudge) != 0 {
+ t.Fatal("the status loop was nudged with the flag off")
+ }
+}
+
+// PF14 (plan 4 R17): runAgent sets the generations file whatever the flag,
+// so a worker's captured serving build is pinned with the flag off too.
+// armAlways publishes the worker's serving version and touches nothing
+// else; armRollout never sets the file either.
+func TestTheFlagOffStillPublishesWorkerArtifacts(t *testing.T) {
+ svc, reg, dd := rolloutFixture(t, false)
+ armAlways(svc, reg, dd)
+ armRollout(context.Background(), noEnv, svc, reg, dd, nil, unusedStore)
+ if svc.OnWorkerServing == nil {
+ t.Fatal("no worker serving publication with the flag off")
+ }
+ if svc.GenerationsFile != "" {
+ t.Fatalf("generations file = %q: armAlways and armRollout must leave it to runAgent", svc.GenerationsFile)
+ }
+}
+
+func TestTheFlagOnArmsTheScheduler(t *testing.T) {
+ svc, reg, dd := rolloutFixture(t, true)
+ svc.GenerationsFile = release.GenerationsPath(dd) // as runAgent sets it, before armRollout
+ _, pub, err := release.GenerateKey()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.Upsert(registry.Route{Host: "a.example.com", Kind: "instance", App: "blog", Account: "bab", Status: "ready",
+ Hibernate: true, Version: "v2"}); err != nil {
+ t.Fatal(err)
+ }
+ store := release.NewDirStore(t.TempDir())
+ ctx, cancel := context.WithCancel(context.Background()) // ends the schedulers' goroutines
+ t.Cleanup(cancel)
+ m := armRollout(ctx, overlapEnv(pub), svc, reg, dd, nil,
+ func(context.Context) (release.Store, error) { return store, nil })
+ if m == nil || armedRollout() != m {
+ t.Fatal("CARLOS_SWAP=overlap did not arm the scheduler")
+ }
+ if svc.Planner == nil || svc.OnGen == nil || svc.OnDrift == nil || svc.CoalesceRecycle == nil {
+ t.Fatal("a hook is missing")
+ }
+ if svc.GenerationsFile != release.GenerationsPath(dd) {
+ t.Fatalf("generations file = %q", svc.GenerationsFile)
+ }
+ // The legs are the scheduler's.
+ if b := adoptBounceFn(svc); b == nil || funcPtr(b) != funcPtr(m.Adopted) {
+ t.Fatal("the adopt bounce is not the scheduler's Adopted")
+ }
+ if fastConvergeLeg(svc, reg) == nil {
+ t.Fatal("no converge leg: the scheduler's pass must run")
+ }
+ cb, ok := rolloutConfigCallbacks(context.Background(), store)
+ if !ok || cb.Delivered == nil || cb.Ack == nil || cb.Restart == nil || cb.Restart("a.example.com") != nil {
+ t.Fatalf("config callbacks = %+v, %v: want the scheduler's delivery and deferred ack, and no restart", cb, ok)
+ }
+ // The pass at arming saw the managed route, and the status probe is
+ // the scheduler's report.
+ r := rolloutStatusProbe()
+ if r == nil || r.Unavailable != "" {
+ t.Fatalf("status = %+v, want the scheduler's report", r)
+ }
+ if got := r.Hosts["a.example.com"]; got.Installed != "v2" {
+ t.Fatalf("status host = %+v, want the managed route installed on v2", got)
+ }
+}
+
+// Spec §6.4, §8.3, §8.4, P2 #1: with the flag on, generations on and the
+// scheduler's state unavailable, managed starts are refused, the reason
+// code reported and the bounce legs stand down. Falling back to today's
+// live-link wakes and legacy convergence (a nil Planner) would exec the
+// failed installed build the unreadable records hold down.
+func TestUnavailableStateRefusesManagedStarts(t *testing.T) {
+ _, pub, err := release.GenerateKey()
+ if err != nil {
+ t.Fatal(err)
+ }
+ dirStore := func(context.Context) (release.Store, error) { return release.NewDirStore(t.TempDir()), nil }
+ for name, tc := range map[string]struct {
+ env func(string) string
+ store func(context.Context) (release.Store, error)
+ book bool // an unreadable failure book in the data dir
+ code string
+ }{
+ "no deploy store": {overlapEnv(pub),
+ func(context.Context) (release.Store, error) { return nil, errors.New("no deployment store") }, false, rolloutNoStore},
+ "no public key": {overlapEnv(""), dirStore, false, rolloutNoReleaseKey},
+ "unreadable book": {overlapEnv(pub), dirStore, true, rolloutBookUnreadable},
+ "book with a bad schema": {overlapEnv(pub), dirStore, true, rolloutBookUnreadable},
+ } {
+ t.Run(name, func(t *testing.T) {
+ svc, reg, dd := rolloutFixture(t, true)
+ if tc.book {
+ body := "{not json"
+ if name == "book with a bad schema" {
+ body = `{"schema": 9}`
+ }
+ if err := os.WriteFile(filepath.Join(dd, rollout.FailureFile), []byte(body), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ m := armRollout(context.Background(), tc.env, svc, reg, dd, nil, tc.store)
+ if m != nil || armedRollout() != nil {
+ t.Fatal("armed without its required state")
+ }
+ if svc.Planner == nil {
+ t.Fatal("no Planner: the box fell back to today's live-link wakes")
+ }
+ managed := registry.Route{Host: "a.example.com", Kind: "instance", App: "blog", Account: "bab", Status: "ready", Hibernate: true, Version: "v2"}
+ p := svc.Planner(context.Background(), managed)
+ if p.Refuse == "" {
+ t.Fatalf("managed wake planned %+v, want refused", p)
+ }
+ if p.Refuse != refuseTexts[tc.code] {
+ t.Fatalf("refusal %q, want the fixed text for %s (never the raw error)", p.Refuse, tc.code)
+ }
+ unit := registry.Route{Host: "u.example.com", Kind: "instance", App: "blog", Account: "bab", Status: "ready", Unit: "blog.service"}
+ if p := svc.Planner(context.Background(), unit); p.Refuse != "" || p.Boot.StagedPath != "" {
+ t.Fatalf("an unmanaged wake was changed: %+v", p)
+ }
+ if r := rolloutStatusProbe(); r == nil || r.Unavailable != tc.code {
+ t.Fatalf("status = %+v, want the reason code %q", r, tc.code)
+ }
+ // The bounce legs stand down.
+ if adoptBounceFn(svc) != nil {
+ t.Fatal("the adopt bounce still stops instances whose next wake is refused")
+ }
+ if fastConvergeLeg(svc, reg) != nil {
+ t.Fatal("legacy convergence still stops instances whose next wake is refused")
+ }
+ if svc.OnDrift == nil {
+ t.Fatal("the sweep's convergence stop is still armed")
+ }
+ if svc.CoalesceRecycle == nil || svc.CoalesceRecycle("bot.internal") {
+ t.Fatal("a worker recycle can still converge onto the live link")
+ }
+ if svc.OnGen != nil {
+ t.Fatal("OnGen set with no scheduler to hear it")
+ }
+ store := release.NewDirStore(t.TempDir())
+ cb, ok := rolloutConfigCallbacks(context.Background(), store)
+ if !ok || cb.Delivered != nil || cb.Restart == nil || cb.Restart("a.example.com") != nil || cb.Ack == nil {
+ t.Fatalf("config callbacks = %+v, %v: want delivery and the ack as today, and no restart", cb, ok)
+ }
+ ctx, s := genKit(t)
+ routes := []registry.Route{execRoute("exec.example.com", "bab", "eleven")}
+ var bounced []string
+ bounce := func(h string) bool { bounced = append(bounced, h); return true }
+ mem := newRestartMemory()
+ askRestart(t, ctx, s, "bab", "eleven")
+ restartRequestLeg(ctx, routes, s, t.TempDir(), bounce, nil, mem)
+ askRestart(t, ctx, s, "bab", "eleven")
+ if !restartRequestLeg(ctx, routes, s, t.TempDir(), bounce, nil, mem) {
+ t.Fatal("the root half of the restart was not asked for")
+ }
+ if len(bounced) != 0 {
+ t.Fatalf("restart bounced %v while managed starts are refused", bounced)
+ }
+ })
+ }
+}
+
+// PF15: with the flag on but no delegated cgroup subtree, the flag is
+// refused before the activator exists. Generations and containment stay
+// off, no manager is armed, no hook is set, every leg is today's, and the
+// status carries the reason code.
+func TestNoDelegationDeclinesTheFlag(t *testing.T) {
+ _, pub, err := release.GenerateKey()
+ if err != nil {
+ t.Fatal(err)
+ }
+ resetRollout(t)
+ eb := &activator.ExecBackend{Control: instancectl.NewClient(filepath.Join(t.TempDir(), "ctl.sock"))}
+ if swapArming(overlapEnv(pub), eb) {
+ t.Fatal("armed generations with no delegated cgroup subtree")
+ }
+ if got := rolloutUnarmed(); got != rolloutNoDelegation {
+ t.Fatalf("declined with %q, want %q", got, rolloutNoDelegation)
+ }
+ // runAgent: SetGenerations(swapOn), then New, then armRollout.
+ svc, reg, dd := rolloutFixture(t, false)
+ declineRollout(rolloutNoDelegation) // rolloutFixture reset it; runAgent would not
+ opened := false
+ m := armRollout(context.Background(), overlapEnv(pub), svc, reg, dd, nil,
+ func(context.Context) (release.Store, error) {
+ opened = true
+ return release.NewDirStore(t.TempDir()), nil
+ })
+ if m != nil || armedRollout() != nil || opened {
+ t.Fatal("armed a scheduler over an activator without generations")
+ }
+ if svc.Planner != nil || svc.OnGen != nil || svc.OnDrift != nil || svc.CoalesceRecycle != nil {
+ t.Fatal("a declined flag set an activator hook")
+ }
+ if r := rolloutStatusProbe(); r == nil || r.Unavailable != rolloutNoDelegation {
+ t.Fatalf("status = %+v, want the reason code %q", r, rolloutNoDelegation)
+ }
+ if rolloutUnavailable() != "" {
+ t.Fatal("a declined flag refuses managed starts")
+ }
+ legsAreTodays(t, svc, reg)
+}
+
+// The armed half of PF15: a delegated subtree and a control channel that
+// speaks the generation verbs arm generations; a channel without them does
+// not.
+func TestDelegationArmsTheFlag(t *testing.T) {
+ _, pub, err := release.GenerateKey()
+ if err != nil {
+ t.Fatal(err)
+ }
+ resetRollout(t)
+ root := t.TempDir()
+ for _, f := range []string{"cgroup.procs", "cgroup.subtree_control"} {
+ if err := os.WriteFile(filepath.Join(root, f), nil, 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ cgm, err := cgroup.SetupAt(root, t.Logf)
+ if err != nil {
+ t.Fatal(err)
+ }
+ eb := &activator.ExecBackend{Cgroups: cgm, Control: instancectl.NewClient(filepath.Join(t.TempDir(), "ctl.sock"))}
+ if !swapArming(overlapEnv(pub), eb) || rolloutUnarmed() != "" {
+ t.Fatalf("delegation present: declined with %q", rolloutUnarmed())
+ }
+ if swapArming(noEnv, eb) {
+ t.Fatal("the flag off armed generations")
+ }
+ eb.Control = nil
+ if swapArming(overlapEnv(pub), eb) || rolloutUnarmed() != rolloutNoGenControl {
+ t.Fatalf("no generation control: armed, or declined with %q", rolloutUnarmed())
+ }
+}
+
+func TestRestartsFeedTheSchedulerWithTheirIdentity(t *testing.T) {
+ routes := []registry.Route{
+ {Host: "a.example.com", Kind: "instance", App: "blog", Account: "bab", Status: "ready", Hibernate: true},
+ {Host: "bot.internal", Kind: "sidecar", App: "blog", Account: "bab", Status: "ready"},
+ {Host: "u.example.com", Kind: "instance", App: "blog", Account: "bab", Status: "ready", Unit: "blog.service"},
+ {Host: "o.example.com", Kind: "instance", App: "other", Account: "bab", Status: "ready", Hibernate: true},
+ }
+ k := [2]string{"bab", "blog"}
+ got := map[string]string{}
+ n := feedRestarts(routes, [][2]string{k}, nil, map[[2]string]string{k: "etag-7"}, func(h, id string) bool { got[h] = id; return true })
+ if n != 2 || got["a.example.com"] != "gen:etag-7" || got["bot.internal"] != "gen:etag-7" {
+ t.Fatalf("fed %d: %v; want the instance and the sidecar, never the named unit or another app", n, got)
+ }
+ scoped := []scopedRestart{{app: k, requests: []release.RestartRequest{
+ {ID: "r1", Scope: release.RestartScope{Kind: release.RestartScopeHost, Value: "a.example.com"}},
+ {ID: "r2", Scope: release.RestartScope{Kind: release.RestartScopeHost, Value: "a.example.com"}},
+ }}}
+ got = map[string]string{}
+ if n := feedRestarts(routes, nil, scoped, nil, func(h, id string) bool { got[h] = id; return true }); n != 1 || got["a.example.com"] != "scope:r2" {
+ t.Fatalf("fed %d: %v; want the narrowed host, identified by the newest request", n, got)
+ }
+}
+
+// Carried obligation 4 (T13): OnChange is a non-blocking nudge that
+// coalesces, so a burst of transitions is one early status tick.
+func TestOnChangeCoalescesIntoOneNudge(t *testing.T) {
+ resetRollout(t)
+ done := make(chan struct{})
+ go func() {
+ for range 100 {
+ nudgeStatus()
+ }
+ close(done)
+ }()
+ select {
+ case <-done:
+ case <-time.After(5 * time.Second):
+ t.Fatal("nudgeStatus blocked")
+ }
+ if len(statusNudge) != 1 {
+ t.Fatalf("%d nudges queued, want one", len(statusNudge))
+ }
+}
+
+// Carried obligation 3 (T11): OnWorkerServing returns at once even while
+// the registry write is slow, and the newest identity a host was handed is
+// the one that lands last.
+func TestTheWorkerServingWriteIsCheapAndKeepsTheNewest(t *testing.T) {
+ release1 := make(chan struct{})
+ var mu sync.Mutex
+ var writes []string
+ w := newServingWriter(func(host, ver string) error {
+ if ver == "v1" {
+ <-release1
+ }
+ mu.Lock()
+ writes = append(writes, host+"="+ver)
+ mu.Unlock()
+ return nil
+ })
+ w.publish("a", "v1") // the drain blocks inside this write
+ start := time.Now()
+ w.publish("a", "v2")
+ w.publish("b", "v1b")
+ w.publish("a", "v3")
+ if d := time.Since(start); d > time.Second {
+ t.Fatalf("publish waited %s on a slow write", d)
+ }
+ close(release1)
+ deadline := time.Now().Add(5 * time.Second)
+ for {
+ w.mu.Lock()
+ idle := !w.running
+ w.mu.Unlock()
+ if idle {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("the writer never drained")
+ }
+ time.Sleep(5 * time.Millisecond)
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ var lastA string
+ for _, x := range writes {
+ if x[:2] == "a=" {
+ lastA = x
+ }
+ }
+ if lastA != "a=v3" || !slices.Contains(writes, "b=v1b") {
+ t.Fatalf("writes %v: want a's newest (v3) last and b's written", writes)
+ }
+ if slices.Contains(writes, "a=v2") {
+ t.Fatalf("writes %v: a stale intermediate version was written", writes)
+ }
+}
+
+// armAlways's callback reaches the registry, through the writer.
+func TestArmAlwaysRecordsTheServingVersion(t *testing.T) {
+ svc, reg, dd := rolloutFixture(t, false)
+ host := "bot.internal"
+ if err := reg.Upsert(registry.Route{Host: host, Kind: "sidecar", App: "blog", Account: "bab", Status: "ready"}); err != nil {
+ t.Fatal(err)
+ }
+ armAlways(svc, reg, dd)
+ svc.OnWorkerServing(host, release.StagedIdentity{Version: "v4"})
+ deadline := time.Now().Add(5 * time.Second)
+ for {
+ if rt, _ := reg.Get(host); rt != nil && rt.ServingVersion == "v4" {
+ return
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("the serving version never reached the registry")
+ }
+ time.Sleep(5 * time.Millisecond)
+ }
+}
+
+// outageStore fails every read the way a bucket outage does.
+type outageStore struct{ release.Store }
+
+func (outageStore) Get(context.Context, string) ([]byte, string, error) {
+ return nil, "", errors.New("503 slow down")
+}
+
+// Carried obligation 1 (T3): the policy loader answers an error for any
+// store failure, and only a confirmed missing object as the defaults. A
+// policy never read is hold (rollout's policyCache).
+func TestThePolicyLoaderFailsOnAnyStoreError(t *testing.T) {
+ ctx := context.Background()
+ if m, err := rolloutPolicy(outageStore{})(ctx, "bab", "blog"); err == nil {
+ t.Fatalf("a store failure read as %+v", m)
+ }
+ s := release.NewDirStore(t.TempDir())
+ m, err := rolloutPolicy(s)(ctx, "bab", "blog")
+ if err != nil {
+ t.Fatalf("a missing config object: %v", err)
+ }
+ if got := rollout.PolicyFromManifest(m); got != rollout.DefaultPolicy() {
+ t.Fatalf("a missing config object read as %+v, want the defaults", got)
+ }
+ if _, err := s.Put(ctx, appconfig.Key("bab", "blog"), []byte("{not json"), ""); err != nil {
+ t.Fatal(err)
+ }
+ if m, err := rolloutPolicy(s)(ctx, "bab", "blog"); err == nil {
+ t.Fatalf("an undecodable config object read as %+v", m)
+ }
+}
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go
index c124c33e1..22ca73b49 100644
--- a/internal/status/snapshot.go
+++ b/internal/status/snapshot.go
@@ -257,9 +257,13 @@ type RolloutReport struct {
Apps map[string]AppRollout `json:"apps,omitempty"` // keyed "<acct>/<app>"
Failures []RolloutFailure `json:"failures,omitempty"`
// Unavailable, when set, is a reason code (not member-facing text) for
- // why CARLOS_SWAP=overlap is on but the scheduler could not arm; every
- // managed start is refused meanwhile (Task 14, refuseManaged). The rest
- // of the report is then empty. Plan 6 words it.
+ // why CARLOS_SWAP=overlap is on but the scheduler could not arm. Either
+ // the box cannot contain generations ("no-cgroup-delegation",
+ // "no-generation-control", "generations-off"), and stays on today's
+ // paths, or its required state is unavailable ("no-deploy-store",
+ // "no-release-key", "failure-book-unreadable"), and every managed start
+ // is refused meanwhile (Task 14, internal/host's swapArming and
+ // refuseManaged). The rest of the report is then empty. Plan 6 words it.
Unavailable string `json:"unavailable,omitempty"`
}