98c1763d6 · carlos/platform

diff --git a/apimode.go b/apimode.go

index 18cbddfc0..f702d3d6e 100644

--- a/apimode.go

+++ b/apimode.go

@@ -3321,7 +3321,10 @@ type apiRollout struct {

HeldBy string `json:"held_by"`

Reason string `json:"reason"`

ConfigPending bool `json:"config_pending"`

- SchedulerIdle bool `json:"scheduler_idle"`

+ // Transient: the box's process for this host is starting or stopping, so

+ // it is neither asleep nor swapped and the deploy watch keeps it pending.

+ Transient bool `json:"transient,omitempty"`

+ SchedulerIdle bool `json:"scheduler_idle"`

// Unavailable is the box's reason code for being unable to arm its

// rollout scheduler. When set the box reports no per-host state: awake

// and scheduler_idle are meaningless false.

diff --git a/deploywatch.go b/deploywatch.go

index 9b0cf298d..1ddf397f9 100644

--- a/deploywatch.go

+++ b/deploywatch.go

@@ -160,6 +160,10 @@ func attemptOpen(a string) bool {

// held_by, which that host carries too.

// - Held by ver, with no failure of its own: a sibling's failure record

// holds it on its good build.

+// - A transient host (its process is starting or stopping) is pending: it

+// is neither asleep nor swapped, and its serving build says nothing

+// settled (Astra R3 section 5 item 10). A failure or a hold it already

+// carries still stands, as above.

// - An attempt in progress is pending whether or not the instance is awake

// (R8): a canary serving ver has not swapped until its settle closes, and

// an asleep instance whose first wake is under observation is not asleep.

@@ -175,6 +179,8 @@ func classifyRollout(r *apiRollout, ver string) rolloutState {

return rolloutFailed

case r.HeldBy == ver:

return rolloutHeld

+ case r.Transient:

+ return rolloutPending

case attemptOpen(r.Attempt):

return rolloutPending

case r.Serving == ver:

diff --git a/deploywatch_test.go b/deploywatch_test.go

index 1b2ca0ece..332dbb0c5 100644

--- a/deploywatch_test.go

+++ b/deploywatch_test.go

@@ -30,6 +30,10 @@ func TestClassifyRollout(t *testing.T) {

{"idle scheduler, B not installed", apiRollout{Installed: "b1", Good: "b1", SchedulerIdle: true}, rolloutPending},

{"swapped", apiRollout{Installed: B, Serving: B, Awake: true}, rolloutSwapped},

{"asleep with B installed", apiRollout{Installed: B, Good: "b1"}, rolloutAsleep},

+ // A process starting or stopping is neither asleep nor swapped.

+ {"transient, idle, no serving build", apiRollout{Installed: B, Good: "b1", Transient: true, SchedulerIdle: true}, rolloutPending},

+ {"transient with B recorded as serving", apiRollout{Installed: B, Serving: B, Awake: true, Transient: true, SchedulerIdle: true}, rolloutPending},

+ {"transient but failed on B", apiRollout{Installed: B, Attempt: status.AttemptDown, AttemptBuild: B, HeldBy: B, Transient: true}, rolloutFailed},

{"queued", apiRollout{Installed: B, Serving: "b1", Awake: true, Attempt: status.AttemptQueued, AttemptBuild: B}, rolloutPending},

{"canary settling", apiRollout{Installed: B, Serving: "b1", Awake: true, Attempt: status.AttemptSettling, AttemptBuild: B}, rolloutPending},

{"failed here", apiRollout{Installed: B, Serving: "b1", Awake: true, Attempt: status.AttemptFailed, AttemptBuild: B, HeldBy: B, Reason: status.ReasonHealth}, rolloutFailed},

@@ -168,6 +172,38 @@ func TestWatchRolloutFollowsTheBoxToDone(t *testing.T) {

}

+// A host whose process is starting or stopping stays pending through a sweep,

+// idle scheduler or not, and completes once it reports asleep or swapped.

+func TestWatchRolloutATransientHostStaysPendingUntilItSettles(t *testing.T) {

+ for _, tc := range []struct {

+ name string

+ final apiRollout

+ want string

+ }{

+ {"asleep", apiRollout{Box: "x1", Installed: B, Good: "b1", SchedulerIdle: true}, watchWordInstalled + " https://a.test (" + watchAsleepTail + ")"},

+ {"swapped", apiRollout{Box: "x1", Installed: B, Serving: B, Awake: true, SchedulerIdle: true}, watchWordSwapped + " https://a.test"},

+ } {

+ t.Run(tc.name, func(t *testing.T) {

+ var out bytes.Buffer

+ var calls atomic.Int64

+ list := sweeps(&calls,

+ []apiInstanceRecord{rec("a.test", &apiRollout{Box: "x1", Installed: B, Good: "b1", Transient: true, SchedulerIdle: true})},

+ []apiInstanceRecord{rec("a.test", &tc.final)},

+ )

+ if err := testWatch(&out, list, noURL(t)).run(targetsFor("a.test")); err != nil {

+ t.Fatalf("watch: %v\n%s", err, out.String())

+ }

+ if calls.Load() < 3 {

+ t.Fatalf("done after %d sweeps, while the host was still starting or stopping", calls.Load())

+ }

+ if !strings.Contains(out.String(), tc.want) {

+ t.Errorf("output lacks %q:\n%s", tc.want, out.String())

+ }

+ })

+ }

+}

+

// R8: a canary already serving B is not swapped while its settle is open.

func TestWatchRolloutCanaryServingBWaitsForItsSettle(t *testing.T) {

var out bytes.Buffer

diff --git a/internal/console/apiinstances.go b/internal/console/apiinstances.go

index 3cebf0275..5bc77a030 100644

--- a/internal/console/apiinstances.go

+++ b/internal/console/apiinstances.go

@@ -355,6 +355,9 @@ type rolloutView struct {

HeldBy string `json:"held_by,omitempty"`

Reason string `json:"reason,omitempty"`

ConfigPending bool `json:"config_pending,omitempty"`

+ // Transient is the box's word that the host's process is starting or

+ // stopping (status.HostRollout.Transient): neither asleep nor awake.

+ Transient bool `json:"transient,omitempty"`

// SchedulerIdle is the box's scheduler for this app: nothing queued,

// running or settling, first-wake observations included (spec §9.1's

// "a box is done" half).

@@ -605,7 +608,7 @@ func (j *boxJoin) foldRollout(b status.Box, override bool) {

put(host, rolloutView{

Box: b.Label, Installed: hr.Installed, Serving: hr.Serving, Good: hr.Good, Awake: hr.Awake,

Attempt: hr.Attempt, AttemptBuild: hr.AttemptBuild, HeldBy: hr.HeldBy, Reason: hr.Reason,

- ConfigPending: hr.ConfigPending, SchedulerIdle: a.Queued == 0 && a.Running == 0 && a.Settling == 0,

+ ConfigPending: hr.ConfigPending, Transient: hr.Transient, SchedulerIdle: a.Queued == 0 && a.Running == 0 && a.Settling == 0,

account: hr.Account, app: hr.App, heldEpoch: hr.HeldEpoch,

})

}

diff --git a/internal/console/rolloutjoin_test.go b/internal/console/rolloutjoin_test.go

index 0dab73be3..c77285386 100644

--- a/internal/console/rolloutjoin_test.go

+++ b/internal/console/rolloutjoin_test.go

@@ -211,3 +211,54 @@ func TestAPIInstanceListSaysWhenTheRolloutJoinIsIncomplete(t *testing.T) {

t.Fatal("a join that skipped an unreadable box snapshot does not say rollout_incomplete")

}

+

+// A box's transient word reaches the member as "transient": true, and an

+// ordinary host carries no such key.

+func TestAPIInstanceListCarriesATransientHost(t *testing.T) {

+ fx, token := instanceFixture(t, "code-inst-transient")

+ enableInstances(t, fx, "titogo", titoManifest())

+ u := instancesURL(fx, "bab", "titogo")

+ for _, h := range []string{"acme.go.tito.io", "beta.go.tito.io"} {

+ if code, _, out := rawReq(t, http.MethodPost, u, token, fmt.Sprintf(`{"host":%q,"region":"ie"}`, h)); code != http.StatusCreated {

+ t.Fatalf("create %s = %d %s", h, code, out)

+ }

+ box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now())

+ box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{

+ Hosts: map[string]status.HostRollout{

+ "acme.go.tito.io": {Account: "bab", App: "titogo", Installed: "v2", Good: "v1", Transient: true},

+ "beta.go.tito.io": {Account: "bab", App: "titogo", Installed: "v2", Serving: "v2", Good: "v2", Awake: true},

+ },

+ Apps: map[string]status.AppRollout{"bab/titogo": {}},

+ }}

+ if err := status.WriteBox(context.Background(), fx.rel, box); err != nil {

+ t.Fatal(err)

+ }

+ code, _, out := rawReq(t, http.MethodGet, u, token, "")

+ if code != http.StatusOK {

+ t.Fatalf("list = %d %s", code, out)

+ }

+ var list struct {

+ Instances []struct {

+ Host string `json:"host"`

+ Rollout json.RawMessage `json:"rollout"`

+ } `json:"instances"`

+ }

+ if err := json.Unmarshal([]byte(out), &list); err != nil {

+ t.Fatal(err)

+ }

+ got := map[string]map[string]any{}

+ for _, r := range list.Instances {

+ var m map[string]any

+ if err := json.Unmarshal(r.Rollout, &m); err != nil {

+ t.Fatalf("%s rollout: %v", r.Host, err)

+ }

+ got[r.Host] = m

+ }

+ if got["acme.go.tito.io"]["transient"] != true {

+ t.Fatalf("acme rollout = %v, want transient true", got["acme.go.tito.io"])

+ }

+ if _, has := got["beta.go.tito.io"]["transient"]; has {

+ t.Fatalf("beta rollout = %v, want no transient key", got["beta.go.tito.io"])

+ }

+}

diff --git a/internal/rollout/status.go b/internal/rollout/status.go

index bfe1f322d..323c780f2 100644

--- a/internal/rollout/status.go

+++ b/internal/rollout/status.go

@@ -6,11 +6,16 @@ import (

// Status is the box's rollout report (spec §8.4, §9.1), for the private

// Metrics block. It reads the queue, the failure book and the config tracks

-// and never calls the activator, so it holds no lock the activator could

-// wait on: Manager.mu is released before any scheduler.mu is taken (the

-// lock order on Manager), and the book is read under a scheduler.mu as the

-// order allows. The hosts' serving state is what the last agent pass and

-// the last attempts recorded, not a fresh ask of the activator.

+// and asks the activator only about a host's process (Owns, Current), with no

+// Manager or scheduler lock held (the lock order on Manager): Manager.mu is

+// released before any scheduler.mu is taken, the host names are gathered under

+// each scheduler.mu and the activator is asked after it is released, and the

+// book is read under a scheduler.mu as the order allows. The hosts' serving

+// state is what the last agent pass and the last attempts recorded, except for

+// a host whose process is starting or stopping (the activator owns it and has

+// no current generation): that one is Transient, with no serving build, because

+// the cached one says nothing settled (Astra R3 section 5 item 10; plan review

+// finding 8).

func (m *Manager) Status() status.RolloutReport {

r := status.RolloutReport{Hosts: map[string]status.HostRollout{}, Apps: map[string]status.AppRollout{}}

for _, f := range m.d.Book.List() {

@@ -42,12 +47,16 @@ func (m *Manager) Status() status.RolloutReport {

}

m.mu.Unlock()

+ transient := m.transientHosts(scheds)

for _, s := range scheds {

app := status.AppRollout{}

s.mu.Lock()

for host, h := range s.q.hosts {

hr := status.HostRollout{Account: s.key.acct, App: s.key.app, Installed: h.wantBuild, Serving: h.serving,

Good: h.good, Awake: h.serving != ""}

+ if transient[host] {

+ hr.Serving, hr.Awake, hr.Transient = "", false, true

+ }

if v, ok := cfg[host]; ok {

hr.ConfigPending, hr.ConfigError = true, v.reason

}

@@ -112,3 +121,36 @@ func (m *Manager) Status() status.RolloutReport {

}

return r

}

+

+// transientHosts is the set of hosts, among the schedulers', whose process is

+// starting or stopping: the activator owns the host and has no current

+// generation for it (a wake or an observation not yet current, a hibernation

+// under way). A host the activator does not own is asleep, and one it has a

+// current generation for is awake; neither is in the set. The names are read

+// under each scheduler.mu and the activator is asked after it is released

+// (scheduler.view's rule), so no Manager or scheduler lock is held across the

+// ask. A host that joins after the names are read is simply not transient in

+// this report; the next one has it.

+func (m *Manager) transientHosts(scheds []*scheduler) map[string]bool {

+ var names []string

+ for _, s := range scheds {

+ s.mu.Lock()

+ for host := range s.q.hosts {

+ names = append(names, host)

+ }

+ s.mu.Unlock()

+ }

+ var out map[string]bool

+ for _, host := range names {

+ if g, ok := m.d.Act.Current(host); ok && g.Build != "" {

+ continue

+ }

+ if m.d.Act.Owns(host) {

+ if out == nil {

+ out = map[string]bool{}

+ }

+ out[host] = true

+ }

+ return out

+}

diff --git a/internal/rollout/status_test.go b/internal/rollout/status_test.go

index 42a9a5a9e..e1d1a3eb9 100644

--- a/internal/rollout/status_test.go

+++ b/internal/rollout/status_test.go

@@ -213,3 +213,91 @@ func TestPassForgetsHostsThatLeftTheBox(t *testing.T) {

return !ok

})

}

+

+// A host whose process is starting or stopping (the activator owns it and has

+// no current generation) is transient: no serving build and not awake, though

+// the last pass cached one. One that is awake or asleep is not (Astra R3

+// section 5 item 10).

+func TestStatusReportsAStartingOrStoppingHostAsTransient(t *testing.T) {

+ h := newHarness(t, "a", "b", "c")

+ h.stage("v2")

+ h.install("v2", "a", "b", "c")

+ h.pass()

+ if got := h.m.Status().Hosts["a"]; got.Transient || got.Serving != "v1" || !got.Awake {

+ t.Fatalf("a before = %+v, want awake on v1 and not transient", got)

+ }

+ h.act.mu.Lock()

+ delete(h.act.gens, "a")

+ h.act.owned["a"] = true // stopping or waking: a process, no current generation

+ delete(h.act.gens, "b") // asleep: owned nowhere

+ h.act.mu.Unlock()

+ // The scheduler's cached serving build is deliberately not refreshed: the

+ // report must not repeat it for a host the activator says is mid-change.

+ r := h.m.Status()

+ a := r.Hosts["a"]

+ if !a.Transient || a.Serving != "" || a.Awake || a.Installed != "v2" {

+ t.Fatalf("a = %+v, want transient, not awake, no serving build", a)

+ }

+ if b := r.Hosts["b"]; b.Transient {

+ t.Fatalf("b = %+v: an asleep host is not transient", b)

+ }

+ if c := r.Hosts["c"]; c.Transient || c.Serving != "v1" || !c.Awake {

+ t.Fatalf("c = %+v: an awake host is not transient", c)

+ }

+}

+

+// Status asks the activator with no Manager or scheduler lock held (the lock

+// order on Manager).

+func TestStatusAsksTheActivatorWithNoLockHeld(t *testing.T) {

+ h := newHarness(t, "a")

+ h.stage("v2")

+ h.install("v2", "a")

+ h.pass()

+ probe := &lockProbeAct{fakeAct: h.act, m: h.m, t: t}

+ h.m.d.Act = probe

+ h.m.Status()

+ if probe.asked.Load() == 0 {

+ t.Fatal("Status never asked the activator")

+ }

+}

+

+// lockProbeAct fails the test when Owns or Current is called while the

+// Manager's or a scheduler's mutex is held.

+type lockProbeAct struct {

+ *fakeAct

+ m *Manager

+ t *testing.T

+ asked atomic.Int64

+}

+

+func (p *lockProbeAct) check() {

+ p.asked.Add(1)

+ if !p.m.mu.TryLock() {

+ p.t.Error("activator asked with Manager.mu held")

+ } else {

+ p.m.mu.Unlock()

+ }

+ p.m.mu.Lock()

+ scheds := make([]*scheduler, 0, len(p.m.apps))

+ for _, s := range p.m.apps {

+ scheds = append(scheds, s)

+ }

+ p.m.mu.Unlock()

+ for _, s := range scheds {

+ if !s.mu.TryLock() {

+ p.t.Error("activator asked with a scheduler.mu held")

+ } else {

+ s.mu.Unlock()

+ }

+}

+

+func (p *lockProbeAct) Owns(host string) bool {

+ p.check()

+ return p.fakeAct.Owns(host)

+}

+

+func (p *lockProbeAct) Current(host string) (activator.Gen, bool) {

+ p.check()

+ return p.fakeAct.Current(host)

+}

diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go

index c6bca2219..fe1cbe0e2 100644

--- a/internal/status/snapshot.go

+++ b/internal/status/snapshot.go

@@ -297,6 +297,12 @@ type HostRollout struct {

// `carlos instances retry` grant, which the box honours only while the

// restriction is unchanged. Absent from an older box, which is 0.

HeldEpoch uint64 `json:"held_epoch,omitempty"`

+ // Transient is set when the activator owns the host's process but has no

+ // current generation for it: the process is starting or stopping, so the

+ // instance is neither asleep nor awake and no serving build is reported.

+ // The deploy watch keeps such a host pending (Astra R3 section 5 item 10).

+ // Absent from an older box, which is false.

+ Transient bool `json:"transient,omitempty"`

}

// AppRollout counts one app's attempts. Settling is canary settles and

carlos / platform Public

zdd-readiness: a host starting or stopping is reported transient and never counted done