carlos / platform Public

zdd-5 task 13: the rollout report in box status

status.RolloutReport and its words ride the private Metrics block only
(dryrun guard fixture added). Manager.Status builds it without calling the
activator; Deps.OnChange nudges after every transition with no lock held;
Manager.observing tracks first-wake observations. reasonWord now returns
the status.Reason* constants (one vocabulary).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

claude 2 weeks ago 0c41859d556a1bc85447ec1b145061aaeda9ecc7 parent 13c5b8c

10 files changed, +443 −30

internal/rollout/events.go +9 −18
internal/rollout/manager.go +34 −4
internal/rollout/manager_test.go +8 −7
internal/rollout/scheduler.go +1 −0
internal/rollout/status.go +86 −0
internal/rollout/status_test.go +163 −0
internal/rollout/wakeplan.go +13 −0
internal/status/dryrun.go +36 −0
internal/status/snapshot.go +82 −1
internal/status/snapshot_test.go +11 −0

diff --git a/internal/rollout/events.go b/internal/rollout/events.go

index 82a4bca82..110873927 100644

--- a/internal/rollout/events.go

+++ b/internal/rollout/events.go

@@ -5,6 +5,7 @@ import (

"time"

"github.com/carlosframework/platform/internal/activator"

+ "github.com/carlosframework/platform/internal/status"

)

// EventCode is a rollout event's stable internal name (spec §9.4). The codes

@@ -24,20 +25,10 @@ const (

EvStagingRefused EventCode = "staging.refused"

)

-// The reason words: the closed vocabulary a member-facing event or status

-// line may give for a failure. Task 13 moves them into the status package

-// (status.Reason*) and points these at them.

-const (

- reasonHealth = "health"

- reasonSettle = "settle"

- reasonExit = "exit"

- reasonStaging = "staging"

- reasonReused = "reused"

// reasonWord turns an attempt's diagnostic text (SwapResult.Reason, an error)

-// into one of the reason words, so no free text, which can name paths and

-// internals, reaches a member-facing surface. The raw text goes to the

+// into one of status's reason words (status.Reason*, the one vocabulary the

+// events and the status report share), so no free text, which can name paths

+// and internals, reaches a member-facing surface. The raw text goes to the

// operator log instead. Anything unrecognised is a failed health check: the

// swap's checks are the health check, the settle and the exit watch, and the

// health check is the one a spawn or start error also fails.

@@ -45,15 +36,15 @@ func reasonWord(text string) string {

t := strings.ToLower(text)

switch {

case strings.Contains(t, "reused"):

- return reasonReused

+ return status.ReasonReused

case strings.Contains(t, "staging"), strings.Contains(t, "staged"), strings.Contains(t, "digest"):

- return reasonStaging

+ return status.ReasonStaging

case strings.Contains(t, "settle"):

- return reasonSettle

+ return status.ReasonSettle

case strings.Contains(t, "exited"):

- return reasonExit

+ return status.ReasonExit

}

- return reasonHealth

+ return status.ReasonHealth

}

func textSwapping(build string, cause activator.Cause) string {

diff --git a/internal/rollout/manager.go b/internal/rollout/manager.go

index c4b3997be..04776166e 100644

--- a/internal/rollout/manager.go

+++ b/internal/rollout/manager.go

@@ -65,8 +65,14 @@ type Deps struct {

// backend's Materialize hook) so a wake is planned against it (Task 10,

// spec §6.4, §8.4). Nil: a cold managed host's wake is refused.

Materialize func(ctx context.Context, host string) error

- Now func() time.Time

- Logf func(format string, args ...any)

+ // OnChange, when set, is told after every rollout transition (an

+ // attempt's outcome, a wake's event, a newly installed build, a retry),

+ // so the box republishes its status within seconds instead of at the

+ // next tick (plan 6's deploy watch). It is a nudge: it is called with no

+ // Manager or scheduler lock held, and it must not block.

+ OnChange func()

+ Now func() time.Time

+ Logf func(format string, args ...any)

}

// pointerReadEvery bounds how often an app with a standing failure record

@@ -112,6 +118,10 @@ type Manager struct {

// nudging holds the hosts a Nudge is working on; true asks it to look

// once more for a nudge that arrived meanwhile.

nudging map[string]bool

+ // observing holds the hosts whose first wake is inside its 10 s

+ // observation: opened by a healthy first wake, closed by whatever ends

+ // it (OnGen). Status reports them as settling.

+ observing map[string]bool

// goodMu serialises every serving- and good-build write with its

// acknowledgement check (served, publishServing, writeGood), so an older

@@ -129,11 +139,19 @@ func NewManager(ctx context.Context, d Deps) *Manager {

}

m := &Manager{d: d, ctx: ctx,

apps: map[appKey]*scheduler{}, hostApp: map[string]appKey{}, cfg: map[string]*configTrack{},

- retried: map[string]string{}, pointerRead: map[appKey]time.Time{}, nudging: map[string]bool{}}

+ retried: map[string]string{}, pointerRead: map[appKey]time.Time{}, nudging: map[string]bool{}, observing: map[string]bool{}}

m.policies = &policyCache{load: d.Policy, ttl: 30 * time.Second, now: d.Now, m: map[appKey]policyEntry{}}

return m

}

+// changed tells OnChange a transition happened. Every caller has released

+// Manager.mu and any scheduler.mu first, so the callback may read Status.

+func (m *Manager) changed() {

+ if m.d.OnChange != nil {

+ m.d.OnChange()

+ }

func (m *Manager) scheduler(k appKey) *scheduler {

m.mu.Lock()

defer m.mu.Unlock()

@@ -248,6 +266,7 @@ func (m *Manager) observe(routes []registry.Route) []appKey {

}

now := m.d.Now()

keys := make([]appKey, 0, len(byApp))

+ moved := false

for k, list := range byApp {

keys = append(keys, k)

policy := m.policies.get(m.ctx, k.acct, k.app)

@@ -278,11 +297,18 @@ func (m *Manager) observe(routes []registry.Route) []appKey {

if retried[x.rt.Host] == x.rt.Version {

cause = activator.CauseRetry

}

- s.q.wantBuild(x.rt.Host, cause, x.rt.Version, now)

+ if s.q.wantBuild(x.rt.Host, cause, x.rt.Version, now) {

+ moved = true

+ }

}

s.mu.Unlock()

s.poke()

}

+ if moved {

+ // A sleeper has no attempt to finish, so the pass that first sees

+ // its new build is the news for the deploy watch.

+ m.changed()

+ }

return keys

}

@@ -383,6 +409,7 @@ func (m *Manager) retryBuild(k appKey, build, host string) {

m.mu.Unlock()

m.d.Logf("rollout: %s/%s build %s retried (%v)", k.acct, k.app, build, hosts)

s.poke()

+ m.changed()

}

// RetryHost is the box half of `carlos instances retry <host>` (spec §8.4):

@@ -498,6 +525,7 @@ func (m *Manager) liftClearedHalts() {

if len(lifted) > 0 {

m.d.Logf("rollout: %s/%s: halt lifted for %v, their failure records are gone", s.key.acct, s.key.app, lifted)

s.poke()

+ m.changed()

}

@@ -533,6 +561,7 @@ func (m *Manager) Restart(host, id string) bool {

s.mu.Unlock()

if changed {

s.poke()

+ m.changed()

}

return changed

}

@@ -576,6 +605,7 @@ func (m *Manager) ConfigDelivered(host, rev string) {

s.mu.Unlock()

if changed {

s.poke()

+ m.changed() // the revision is delivered and not yet active, which Status shows

}

diff --git a/internal/rollout/manager_test.go b/internal/rollout/manager_test.go

index df1b8617f..183231c24 100644

--- a/internal/rollout/manager_test.go

+++ b/internal/rollout/manager_test.go

@@ -11,6 +11,7 @@ import (

"github.com/carlosframework/platform/internal/appconfig"

"github.com/carlosframework/platform/internal/registry"

"github.com/carlosframework/platform/internal/release"

+ "github.com/carlosframework/platform/internal/status"

)

func TestAReleaseRollsOutCanaryFirstThenPaced(t *testing.T) {

@@ -933,13 +934,13 @@ func TestMemberEventsCarryOnlyTheReasonWord(t *testing.T) {

func TestReasonWords(t *testing.T) {

for text, want := range map[string]string{

- "health check failed": reasonHealth,

- "swap: successor exited during its settle": reasonSettle,

- "exited within 10s of becoming healthy": reasonExit,

- "staging: digest mismatch": reasonStaging,

- "version name reused with different content, ship under a new name": reasonReused,

- "GET / answered 500": reasonHealth,

- "": reasonHealth,

+ "health check failed": status.ReasonHealth,

+ "swap: successor exited during its settle": status.ReasonSettle,

+ "exited within 10s of becoming healthy": status.ReasonExit,

+ "staging: digest mismatch": status.ReasonStaging,

+ "version name reused with different content, ship under a new name": status.ReasonReused,

+ "GET / answered 500": status.ReasonHealth,

+ "": status.ReasonHealth,

} {

if got := reasonWord(text); got != want {

t.Errorf("reasonWord(%q) = %q, want %q", text, got, want)

diff --git a/internal/rollout/scheduler.go b/internal/rollout/scheduler.go

index 402be5e23..8914a46f0 100644

--- a/internal/rollout/scheduler.go

+++ b/internal/rollout/scheduler.go

@@ -221,6 +221,7 @@ func (s *scheduler) done(d dispatch, res activator.SwapResult, mode activator.Sw

}

s.mu.Unlock()

defer s.poke()

+ defer m.changed() // runs first: the slot is free and the record is written

if recErr != nil {

m.d.Logf("rollout: write failure record %s/%s %s: %v (held in memory)", rec.Account, rec.App, rec.Build, recErr)

}

diff --git a/internal/rollout/status.go b/internal/rollout/status.go

new file mode 100644

index 000000000..5e938b5f9

--- /dev/null

+++ b/internal/rollout/status.go

@@ -0,0 +1,86 @@

+package rollout

+import (

+ "github.com/carlosframework/platform/internal/status"

+// Status is the box's rollout report (spec §8.4, §9.1), for the private

+// Metrics block. It reads the queue, the failure book and the config tracks

+// and never calls the activator, so it holds no lock the activator could

+// wait on: Manager.mu is released before any scheduler.mu is taken (the

+// lock order on Manager), and the book is read under a scheduler.mu as the

+// order allows. The hosts' serving state is what the last agent pass and

+// the last attempts recorded, not a fresh ask of the activator.

+func (m *Manager) Status() status.RolloutReport {

+ r := status.RolloutReport{Hosts: map[string]status.HostRollout{}, Apps: map[string]status.AppRollout{}}

+ for _, f := range m.d.Book.List() {

+ r.Failures = append(r.Failures, status.RolloutFailure{Account: f.Account, App: f.App, Build: f.Build, Host: f.Host,

+ Mode: f.Mode, Reason: reasonWord(f.Reason), PromotedAt: f.PromotedAt, At: f.At, HeldDown: f.HeldDown})

+ }

+ m.mu.Lock()

+ scheds := make([]*scheduler, 0, len(m.apps))

+ for _, s := range m.apps {

+ scheds = append(scheds, s)

+ }

+ observing := make(map[string]bool, len(m.observing))

+ for h := range m.observing {

+ observing[h] = true

+ }

+ type cfgView struct {

+ pending bool

+ reason string

+ }

+ cfg := make(map[string]cfgView, len(m.cfg))

+ for h, t := range m.cfg {

+ if t.delivered != "" && t.delivered != t.active {

+ v := cfgView{pending: true}

+ if t.lastErr != "" {

+ // A reason word, never the raw text (which can name paths).

+ v.reason = reasonWord(t.lastErr)

+ }

+ cfg[h] = v

+ }

+ m.mu.Unlock()

+ for _, s := range scheds {

+ app := status.AppRollout{}

+ s.mu.Lock()

+ for host, h := range s.q.hosts {

+ hr := status.HostRollout{Account: s.key.acct, App: s.key.app, Installed: h.wantBuild, Serving: h.serving,

+ Good: h.good, Awake: h.serving != ""}

+ if v, ok := cfg[host]; ok {

+ hr.ConfigPending, hr.ConfigError = true, v.reason

+ }

+ switch {

+ case h.running != nil && (h.running.Canary || observing[host]):

+ hr.Attempt, hr.AttemptBuild = status.AttemptSettling, h.running.ID.Build

+ app.Settling++

+ case h.running != nil:

+ hr.Attempt, hr.AttemptBuild = status.AttemptRunning, h.running.ID.Build

+ app.Running++

+ case observing[host]:

+ hr.Attempt, hr.AttemptBuild = status.AttemptSettling, h.wantBuild

+ app.Settling++

+ default:

+ if d, ok := s.q.pending(h); ok {

+ hr.Attempt, hr.AttemptBuild, app.Queued = status.AttemptQueued, d.ID.Build, app.Queued+1

+ }

+ // A build held for this instance, with nothing else going on: the

+ // failed words. A held-down instance is down.

+ if hr.Attempt == "" && h.wantBuild != "" && h.serving != h.wantBuild {

+ if rec, ok := m.d.Book.Lookup(s.key.acct, s.key.app, h.wantBuild); ok &&

+ m.d.Book.Blocks(s.key.acct, s.key.app, h.wantBuild, host, h.wantBuild == h.good) {

+ hr.Attempt, hr.AttemptBuild, hr.HeldBy, hr.Reason = status.AttemptFailed, rec.Build, rec.Build, reasonWord(rec.Reason)

+ if m.d.Book.HeldDown(s.key.acct, s.key.app, h.wantBuild, host) {

+ hr.Attempt = status.AttemptDown

+ }

+ r.Hosts[host] = hr

+ }

+ s.mu.Unlock()

+ r.Apps[s.key.acct+"/"+s.key.app] = app

+ }

+ return r

diff --git a/internal/rollout/status_test.go b/internal/rollout/status_test.go

new file mode 100644

index 000000000..57ecb6797

--- /dev/null

+++ b/internal/rollout/status_test.go

@@ -0,0 +1,163 @@

+package rollout

+import (

+ "sync/atomic"

+ "testing"

+ "github.com/carlosframework/platform/internal/activator"

+ "github.com/carlosframework/platform/internal/release"

+ "github.com/carlosframework/platform/internal/status"

+func TestStatusReportsHeldAndRollingInstances(t *testing.T) {

+ h := newHarness(t, "a", "b")

+ h.stage("v2")

+ gate := h.act.hold("a")

+ h.install("v2", "a", "b")

+ h.pass()

+ waitFor(t, "a's canary", func() bool { _, s, _ := h.act.snapshot(); return len(s) == 1 })

+ // Recorded after the pass: a pass prunes the records of builds no route

+ // installs, and v3 is installed nowhere.

+ if _, err := h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v3", Host: "z", Reason: "health check failed", At: t0}); err != nil {

+ t.Fatal(err)

+ }

+ r := h.m.Status()

+ if got := r.Hosts["a"]; got.Attempt != status.AttemptSettling || got.AttemptBuild != "v2" || got.Installed != "v2" || got.Serving != "v1" || !got.Awake {

+ t.Fatalf("a = %+v, want the canary settling onto v2", got)

+ }

+ if got := r.Hosts["b"]; got.Attempt != status.AttemptQueued || got.AttemptBuild != "v2" {

+ t.Fatalf("b = %+v, want queued behind the canary", got)

+ }

+ if app := r.Apps["bab/blog"]; app.Settling != 1 || app.Queued != 1 || app.Running != 0 {

+ t.Fatalf("app = %+v", app)

+ }

+ if len(r.Failures) != 1 || r.Failures[0].Build != "v3" || r.Failures[0].Reason != status.ReasonHealth {

+ t.Fatalf("failures = %+v", r.Failures)

+ }

+ close(gate)

+func TestAFailedFirstWakeShowsAsHeld(t *testing.T) {

+ h := newHarness(t, "a")

+ h.act.mu.Lock()

+ delete(h.act.gens, "a") // asleep: v2 is a first wake's build

+ h.act.mu.Unlock()

+ h.stage("v2")

+ h.install("v2", "a")

+ h.pass()

+ h.act.serve("a", "v1") // the first wake failed and the good build serves in its place

+ cur, _ := h.act.Current("a")

+ h.m.OnGen(activator.GenEvent{Kind: activator.GenFellBack, Route: h.rows.get("a"), FirstWake: true,

+ Mode: activator.ModeOverlap, Gen: cur, Failed: release.StagedIdentity{Version: "v2"},

+ Reason: "exited within 10s of becoming healthy"})

+ h.pass()

+ got := h.m.Status().Hosts["a"]

+ if got.Attempt != status.AttemptFailed || got.AttemptBuild != "v2" || got.HeldBy != "v2" || got.Reason != status.ReasonExit || got.Serving != "v1" {

+ t.Fatalf("a = %+v", got)

+ }

+// A hold build that failed on the instance is held down: its wakes are

+// refused, which the report calls down.

+func TestAHeldDownInstanceShowsAsDown(t *testing.T) {

+ h := newHarness(t, "a")

+ h.stage("v2")

+ if _, err := h.book.Record(FailureRecord{Account: "bab", App: "blog", Build: "v2", Host: "a", Mode: "hold",

+ Reason: "swap: successor exited during its settle", At: t0, HeldDown: []string{"a"}}); err != nil {

+ t.Fatal(err)

+ }

+ h.install("v2", "a")

+ h.pass()

+ got := h.m.Status().Hosts["a"]

+ if got.Attempt != status.AttemptDown || got.HeldBy != "v2" || got.Reason != status.ReasonSettle {

+ t.Fatalf("a = %+v", got)

+ }

+// Spec §6.4, §9.1: an intentional stop (hibernation, maintenance) during a

+// first wake's 10 s ends the observation with no verdict. GenCancelled closes

+// what GenHealthy opened, so the host stops showing as settling and the

+// deploy watch can complete; nothing is recorded against the build. With no

+// closing event, observing[host] stayed set for ever.

+func TestACancelledObservationStopsSettlingAndRecordsNothing(t *testing.T) {

+ h := newHarness(t, "a")

+ h.act.mu.Lock()

+ delete(h.act.gens, "a") // asleep: its first wake is the only thing that runs v2

+ h.act.mu.Unlock()

+ h.stage("v2")

+ h.install("v2", "a")

+ h.pass()

+ g := activator.Gen{N: 9, Build: "v2", Digest: "sha-v2"}

+ h.m.OnGen(activator.GenEvent{Kind: activator.GenHealthy, Route: h.rows.get("a"), Gen: g, FirstWake: true})

+ if got := h.m.Status().Hosts["a"]; got.Attempt != status.AttemptSettling {

+ t.Fatalf("under observation: a = %+v, want settling", got)

+ }

+ h.m.OnGen(activator.GenEvent{Kind: activator.GenCancelled, Route: h.rows.get("a"), Gen: g, FirstWake: true})

+ if got := h.m.Status().Hosts["a"]; got.Attempt == status.AttemptSettling {

+ t.Fatalf("after the cancelled observation: a = %+v, still settling", got)

+ }

+ if _, ok := h.book.Lookup("bab", "blog", "v2"); ok {

+ t.Fatal("a cancelled observation was recorded as a failure of the build")

+ }

+ if r := h.rows.get("a"); r.GoodVersion != "v1" {

+ t.Fatalf("good build = %q: a cancelled observation is no settle", r.GoodVersion)

+ }

+// Plan 6's deploy watch counts a sleeper as "installed, awaiting first

+// wake" once its box reports B installed. No attempt runs for a sleeper, so

+// the pass that first sees B must republish.

+func TestANewlyInstalledBuildIsPublishedAtOnce(t *testing.T) {

+ h := newHarness(t, "a")

+ var changes atomic.Int32

+ h.m.d.OnChange = func() { changes.Add(1) }

+ h.act.mu.Lock()

+ delete(h.act.gens, "a") // asleep

+ h.act.mu.Unlock()

+ h.stage("v2")

+ h.install("v2", "a")

+ h.pass()

+ if changes.Load() == 0 {

+ t.Fatal("a newly installed build waited for the five-minute status tick")

+ }

+ if got := h.m.Status().Hosts["a"]; got.Installed != "v2" || got.Awake {

+ t.Fatalf("a = %+v, want v2 installed and asleep", got)

+ }

+// OnChange is a nudge called after every transition, with no manager or

+// scheduler lock held: this one reads Status, which takes both, so a call

+// under either lock deadlocks the test.

+func TestOnChangeFollowsEveryTransitionWithNoLockHeld(t *testing.T) {

+ h := newHarness(t, "a")

+ var changes atomic.Int32

+ h.m.d.OnChange = func() {

+ h.m.Status()

+ changes.Add(1)

+ }

+ h.stage("v2")

+ h.install("v2", "a")

+ h.pass()

+ waitFor(t, "a on v2", func() bool { return h.servingAll("v2", "a") })

+ waitFor(t, "the attempt's change", func() bool { return changes.Load() >= 2 }) // the pass's, then done()'s

+ before := changes.Load()

+ h.m.OnGen(activator.GenEvent{Kind: activator.GenSettled, Route: h.rows.get("a"), Gen: activator.Gen{N: 1, Build: "v2"}})

+ if changes.Load() <= before {

+ t.Fatal("OnGen did not report its transition")

+ }

+func TestStatusReportsAConfigRevisionDeliveredButNotActive(t *testing.T) {

+ h := newHarness(t, "a")

+ h.pass()

+ gate := h.act.hold("a")

+ h.m.ConfigDelivered("a", "rev1")

+ waitFor(t, "the config attempt", func() bool { _, s, _ := h.act.snapshot(); return len(s) == 1 })

+ got := h.m.Status().Hosts["a"]

+ // The config attempt is its cohort's canary, so it reads as settling.

+ if !got.ConfigPending || got.Attempt != status.AttemptSettling || got.AttemptBuild != "v1" || got.ConfigError != "" {

+ t.Fatalf("a = %+v, want the config attempt in flight with its revision pending", got)

+ }

+ close(gate)

+ waitFor(t, "the config to activate", func() bool { return !h.m.Status().Hosts["a"].ConfigPending })

diff --git a/internal/rollout/wakeplan.go b/internal/rollout/wakeplan.go

index c7f66911c..099651ae2 100644

--- a/internal/rollout/wakeplan.go

+++ b/internal/rollout/wakeplan.go

@@ -196,6 +196,19 @@ func (m *Manager) attemptRunning(host string) bool {

// they take goodMu and call back into Current.

func (m *Manager) OnGen(ev activator.GenEvent) {

rt := ev.Route

+ m.mu.Lock()

+ switch {

+ case ev.Kind == activator.GenHealthy && ev.FirstWake:

+ m.observing[rt.Host] = true

+ case ev.Kind != activator.GenHealthy:

+ // GenSettled, GenFailed, GenFellBack and GenCancelled (an

+ // intentional stop ended the observation with no verdict) all close

+ // it. The switch below has no recording arm for GenCancelled, so a

+ // cancelled observation records nothing (spec §6.4).

+ delete(m.observing, rt.Host)

+ }

+ m.mu.Unlock()

+ defer m.changed() // after the event's own work, with no lock held

switch ev.Kind {

case activator.GenHealthy:

// A wake that is not a first wake includes the good build booted

diff --git a/internal/status/dryrun.go b/internal/status/dryrun.go

index 377af3410..83dc32d8e 100644

--- a/internal/status/dryrun.go

+++ b/internal/status/dryrun.go

@@ -132,6 +132,14 @@ const dryRunProofWaitHost = "proof-wait.internal"

var dryRunProofWaitNeedles = []string{dryRunProofWaitHost}

+// dryRunRolloutHost is the rollout report needle: Box.Metrics.Rollout names

+// the instances the rollout scheduler manages (hosts, failure records), so

+// the same scope split applies. REQUIRED in status/boxes/*.json and BANNED in

+// every other published object.

+const dryRunRolloutHost = "rollout-host.internal"

+var dryRunRolloutNeedles = []string{dryRunRolloutHost}

// ScanForLeaks reports which of banned's strings appear anywhere in j. It is

// the single implementation both DryRun's self-check and

// contract_test.go's scanForLeaks test helper run — the test package can't

@@ -227,6 +235,17 @@ func dryRunFixtures() ([]registry.Route, Probes) {

ProofWaits: []ProofWait{

{Host: dryRunProofWaitHost, Version: "v-dryrun", SHA256: "dryrunproofsha", Prover: "dev", Reason: "no proof from dev", Since: time.Now()},

+ // One held instance and its failure record, for the rollout scope check.

+ Rollout: &RolloutReport{

+ Hosts: map[string]HostRollout{

+ dryRunRolloutHost: {Account: "dryrun", App: "app", Installed: "v-dryrun", Serving: "v-old", Good: "v-old", Awake: true,

+ Attempt: AttemptFailed, AttemptBuild: "v-dryrun", HeldBy: "v-dryrun", Reason: ReasonHealth},

+ },

+ Apps: map[string]AppRollout{"dryrun/app": {Queued: 1}},

+ Failures: []RolloutFailure{

+ {Account: "dryrun", App: "app", Build: "v-dryrun", Host: dryRunRolloutHost, Mode: "overlap", Reason: ReasonHealth, PromotedAt: time.Now(), At: time.Now()},

+ },

// The probe table, keyed by a route host and carrying a backend

// address — see dryRunUpstreamNeedles. One up and one down upstream,

// so the down one's reason string is exercised too.

@@ -388,6 +407,9 @@ func DryRun(dir string, out io.Writer) error {

// proofWaitKeySeen is the same positive half for D3's proof wait list

// (dryRunProofWaitNeedles).

proofWaitKeySeen := false

+ // rolloutKeySeen is the same positive half for the rollout report

+ // (dryRunRolloutNeedles).

+ rolloutKeySeen := false

for _, key := range keys {

body, _, err := s.Get(ctx, key)

@@ -480,6 +502,17 @@ func DryRun(dir string, out io.Writer) error {

return fmt.Errorf("dry run: %s leaked a proof_waits fixture %v — ProofWaits must never reach the public feed", key, leaks)

}

+ // dryRunRolloutNeedles: the seventh scope split, on the same

+ // pattern. The rollout report names the managed instances.

+ if strings.HasPrefix(key, "status/boxes/") {

+ if leaks := ScanForLeaks(body, dryRunRolloutNeedles); len(leaks) == 0 {

+ return fmt.Errorf("dry run: %s: expected rollout fixtures %v, found none — Rollout not attached?", key, dryRunRolloutNeedles)

+ }

+ rolloutKeySeen = true

+ } else if leaks := ScanForLeaks(body, dryRunRolloutNeedles); len(leaks) > 0 {

+ return fmt.Errorf("dry run: %s leaked a rollout fixture %v — Rollout must never reach the public feed", key, leaks)

+ }

// Page assets (index.html, status.css) aren't JSON and simply fail

// to decode here — silently skipped, since the band-digit check

// only means anything for objects that actually carry Load/Capacity

@@ -513,6 +546,9 @@ func DryRun(dir string, out io.Writer) error {

if !proofWaitKeySeen {

return fmt.Errorf("dry run: no status/boxes/*.json object was scanned — proof_waits scope check never ran")

}

+ if !rolloutKeySeen {

+ return fmt.Errorf("dry run: no status/boxes/*.json object was scanned — rollout scope check never ran")

+ }

fmt.Fprintln(out, "dry-run: no leaks")

return nil

diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go

index e022884e9..c124c33e1 100644

--- a/internal/status/snapshot.go

+++ b/internal/status/snapshot.go

@@ -172,6 +172,9 @@ type Probes struct {

// ProofWaits is the hosts held on their current build waiting for a dev

// proof (D3). Private Metrics only: they name hosts.

ProofWaits []ProofWait

+ // Rollout is the rollout scheduler's report (zero-downtime spec §8.4,

+ // §9.1); nil with CARLOS_SWAP off. Private Metrics only: it names hosts.

+ Rollout *RolloutReport

// ConfigGuard is "" in the normal mode and a words-only note when the

// config guard's explicit bypass is in effect on this box (amadan #213).

ConfigGuard string

@@ -246,6 +249,79 @@ type ReleaseRefusal struct {

RecoveryError string `json:"recovery_error,omitempty"`

}

+// RolloutReport is this box's rollout scheduler state (zero-downtime spec

+// §8.4, §9.1), published only with CARLOS_SWAP=overlap. It names hosts, so

+// it lives in the private Metrics block alone.

+type RolloutReport struct {

+ Hosts map[string]HostRollout `json:"hosts,omitempty"`

+ Apps map[string]AppRollout `json:"apps,omitempty"` // keyed "<acct>/<app>"

+ Failures []RolloutFailure `json:"failures,omitempty"`

+ // Unavailable, when set, is a reason code (not member-facing text) for

+ // why CARLOS_SWAP=overlap is on but the scheduler could not arm; every

+ // managed start is refused meanwhile (Task 14, refuseManaged). The rest

+ // of the report is then empty. Plan 6 words it.

+ Unavailable string `json:"unavailable,omitempty"`

+// HostRollout is one managed instance. Attempt and Reason are the closed

+// words below; plan 6 renders them.

+type HostRollout struct {

+ Account string `json:"account"`

+ App string `json:"app"`

+ Installed string `json:"installed"`

+ Serving string `json:"serving,omitempty"`

+ Good string `json:"good,omitempty"`

+ Awake bool `json:"awake"`

+ Attempt string `json:"attempt,omitempty"`

+ AttemptBuild string `json:"attempt_build,omitempty"`

+ HeldBy string `json:"held_by,omitempty"`

+ Reason string `json:"reason,omitempty"`

+ ConfigPending bool `json:"config_pending,omitempty"`

+ // ConfigError is a reason word (Reason*), never raw error text.

+ ConfigError string `json:"config_error,omitempty"`

+// AppRollout counts one app's attempts. Settling is canary settles and

+// first-wake observations under way.

+type AppRollout struct {

+ Queued int `json:"queued"`

+ Running int `json:"running"`

+ Settling int `json:"settling"`

+// RolloutFailure is one failure record (spec §8.4). Reason is a reason word.

+type RolloutFailure struct {

+ Account string `json:"account"`

+ App string `json:"app"`

+ Build string `json:"build"`

+ Host string `json:"host"`

+ Mode string `json:"mode,omitempty"`

+ Reason string `json:"reason"`

+ PromotedAt time.Time `json:"promoted_at"`

+ At time.Time `json:"at"`

+ HeldDown []string `json:"held_down,omitempty"`

+// The attempt words.

+const (

+ AttemptQueued = "queued"

+ AttemptRunning = "running"

+ AttemptSettling = "settling"

+ AttemptFailed = "failed"

+ AttemptDown = "down"

+// The reason words: the closed vocabulary a member-facing event or status

+// line may give for a failure. internal/rollout maps every diagnostic text

+// onto one of them (reasonWord).

+const (

+ ReasonHealth = "health"

+ ReasonSettle = "settle"

+ ReasonExit = "exit"

+ ReasonStaging = "staging"

+ ReasonReused = "reused"

// ProofWait is a host this box is holding on its current build because its

// channel head has no dev proof yet (D3, release.ProofGate), or because the

// box's proof policy file does not load. PRIVATE like the rest of

@@ -425,6 +501,10 @@ type BoxMetrics struct {

// until a dev proof for the new head arrives (D3). Hostname-bearing, so

// it lives in this private block only. Omitted when empty.

ProofWaits []ProofWait `json:"proof_waits,omitempty"`

+ // Rollout is the rollout scheduler's report (zero-downtime spec §8.4,

+ // §9.1). Hostname-bearing, so it lives in this private block only.

+ // Omitted when the scheduler is off.

+ Rollout *RolloutReport `json:"rollout,omitempty"`

// ConfigGuard says the box's admission guard is bypassed, e.g.

// "bypassed (CARLOS_NO_SEALED_CONFIG)": it admits placed exec instances

// without checking their sealed config. Omitted in the normal mode, so an

@@ -823,7 +903,7 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box {

p.DiskFreeBytes != nil || p.Release != "" || p.Tunnel != "" ||

p.InstanceID != "" || p.Platform != "" || p.SystemChannel != "" || p.System != nil ||

len(routeVersions) > 0 || len(aliasReadings) > 0 || len(p.Upstreams) > 0 ||

- len(routeDBBytes) > 0 || p.Layout != nil || len(p.ConfigRefusals) > 0 || len(p.ReleaseRefusals) > 0 || len(p.ProofWaits) > 0 || p.ConfigGuard != "" ||

+ len(routeDBBytes) > 0 || p.Layout != nil || len(p.ConfigRefusals) > 0 || len(p.ReleaseRefusals) > 0 || len(p.ProofWaits) > 0 || p.Rollout != nil || p.ConfigGuard != "" ||

p.Alerts != nil || p.GeoIP != nil {

box.Metrics = &BoxMetrics{

ConsoleProcesses: p.ConsoleProcesses,

@@ -859,6 +939,7 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box {

ConfigRefusalsTotal: p.ConfigRefusalsTotal,

ReleaseRefusals: append([]ReleaseRefusal(nil), p.ReleaseRefusals...),

ProofWaits: append([]ProofWait(nil), p.ProofWaits...),

+ Rollout: p.Rollout,

ConfigGuard: p.ConfigGuard,

}

diff --git a/internal/status/snapshot_test.go b/internal/status/snapshot_test.go

index 85fdcb47c..1afd4c3b7 100644

--- a/internal/status/snapshot_test.go

+++ b/internal/status/snapshot_test.go

@@ -157,3 +157,14 @@ func TestCollectBoxCarriesGeoIP(t *testing.T) {

t.Fatal("no GeoIP and nothing else known must still mean no metrics block")

}

+func TestTheRolloutReportRidesThePrivateMetrics(t *testing.T) {

+ r := &RolloutReport{Hosts: map[string]HostRollout{"a.example.com": {App: "blog", Installed: "v2", Serving: "v1", Attempt: AttemptFailed, HeldBy: "v2", Reason: ReasonHealth}}}

+ box := CollectBox(nil, Probes{Rollout: r}, time.Now())

+ if box.Metrics == nil || box.Metrics.Rollout == nil || box.Metrics.Rollout.Hosts["a.example.com"].HeldBy != "v2" {

+ t.Fatalf("metrics = %+v", box.Metrics)

+ }

+ if b := CollectBox(nil, Probes{}, time.Now()); b.Metrics != nil {

+ t.Fatal("the flag off publishes no rollout block and no Metrics because of it")

+ }