amadan

carlos / platform Public

zdd-6 task 8: rollout state on the instance list; held_until on promote

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude eca3d3a3dfaece51baa67e3999992f95b2fbb7f7 parent d0996d3
4 files changed, +283 −10
  • internal/console/apicli.go +24 −7
  • internal/console/apiinstances.go +85 −2
  • internal/console/rolloutjoin_test.go +173 −0
  • internal/provision/plan9_b8_census_test.go +1 −1
diff --git a/internal/console/apicli.go b/internal/console/apicli.go
index 8b95788a4..b5c75ea34 100644
--- a/internal/console/apicli.go
+++ b/internal/console/apicli.go
@@ -1006,9 +1006,10 @@ func (s *Server) handleAPIPromote(w http.ResponseWriter, r *http.Request) {
// sends the client off to watch for an adopt that is a day away —
// exactly the false failure this field exists to prevent. Nothing is
// announced here (no bump): the pointer did not move.
- writeJSON(w, http.StatusOK, map[string]any{
- "result": "noop", "held": s.channelHeld(ctx, pipe, acct.Sqid, app, channel),
- })
+ heldNow := s.channelHeld(ctx, pipe, acct.Sqid, app, channel)
+ out := map[string]any{"result": "noop", "held": heldNow}
+ s.addHeldUntil(ctx, out, heldNow, pipe, acct.Sqid, app, channel)
+ writeJSON(w, http.StatusOK, out)
return
}
s.auditActivity(acct, app, addr, Activity{
@@ -1016,10 +1017,26 @@ func (s *Server) handleAPIPromote(w http.ResponseWriter, r *http.Request) {
Detail: "promoted " + version + " to " + channel + " (cli)",
})
s.bumpSerial(ctx)
- writeJSON(w, http.StatusOK, map[string]any{
- "result": "promoted", "from": res.From, "to": res.To, "noop": false,
- "held": s.channelHeld(ctx, pipe, acct.Sqid, app, channel),
- })
+ heldNow := s.channelHeld(ctx, pipe, acct.Sqid, app, channel)
+ out := map[string]any{"result": "promoted", "from": res.From, "to": res.To, "noop": false, "held": heldNow}
+ s.addHeldUntil(ctx, out, heldNow, pipe, acct.Sqid, app, channel)
+ writeJSON(w, http.StatusOK, out)
+}
+
+// addHeldUntil says when a held promotion lifts (spec §9.1: "held: a bake
+// hold, with the time it lifts"): the pointer's promotion time plus the
+// channel's window. A box starts its window when it first sees the build, a
+// few seconds later, so the CLI says "about". Omitted when nothing is held or
+// the pointer will not read; the CLI then prints its old line.
+func (s *Server) addHeldUntil(ctx context.Context, out map[string]any, held bool, p *release.Pipeline, acct, app, channel string) {
+ if !held {
+ return
+ }
+ cur, err := release.Channel(ctx, s.cfg.Releases, s.cfg.ReleasePub, acct, app, channel)
+ if err != nil {
+ return
+ }
+ out["held_until"] = cur.PromotedAt.Add(holdWindow(p, channel)).UTC().Format(time.RFC3339)
}
// channelHeld answers the CLI's next question without a second round trip: is
diff --git a/internal/console/apiinstances.go b/internal/console/apiinstances.go
index 06218c128..36f82cf3a 100644
--- a/internal/console/apiinstances.go
+++ b/internal/console/apiinstances.go
@@ -195,6 +195,11 @@ type instanceListRecord struct {
// box has sized it: not minted yet, no database file yet, or a route
// whose database lives somewhere the platform does not know about.
DBBytes *int64 `json:"db_bytes,omitempty"`
+ // Rollout is this host's rollout as its box last reported it
+ // (zero-downtime spec §9.1), joined under StagedVersion's staleness rule.
+ // Absent when no fresh box reports one: a box from before plan 5, one
+ // with CARLOS_SWAP off, or one gone dark. The deploy watch reads it.
+ Rollout *rolloutView `json:"rollout,omitempty"`
}
// instanceHealth computes each record host's health word the same way the
@@ -333,6 +338,45 @@ type boxJoin struct {
versions map[string]string
upstreams map[string][]status.UpstreamState
dbBytes map[string]int64
+ rollouts map[string]rolloutView
+}
+
+// rolloutView is one host's rollout as a member sees it. account and app are
+// the report's own, kept so the join can refuse to cross tenants (rollout).
+type rolloutView struct {
+ Box string `json:"box"`
+ Installed string `json:"installed"`
+ Serving string `json:"serving,omitempty"`
+ Good string `json:"good,omitempty"`
+ Awake bool `json:"awake"`
+ Attempt string `json:"attempt,omitempty"`
+ AttemptBuild string `json:"attempt_build,omitempty"`
+ HeldBy string `json:"held_by,omitempty"`
+ Reason string `json:"reason,omitempty"`
+ ConfigPending bool `json:"config_pending,omitempty"`
+ // SchedulerIdle is the box's scheduler for this app: nothing queued,
+ // running or settling, first-wake observations included (spec §9.1's
+ // "a box is done" half).
+ SchedulerIdle bool `json:"scheduler_idle"`
+ // Unavailable is the reason code a box gives for being unable to arm its
+ // rollout scheduler (RolloutReport.Unavailable). When set, the view is
+ // only Box, Installed and this: the box reports no per-host state.
+ Unavailable string `json:"unavailable,omitempty"`
+ account, app string
+ // refused marks a view built from an Unavailable box, which has no
+ // account or app to check (hosts are unique; the caller filtered by app).
+ refused bool
+}
+
+// rollout is the join's answer for host, or nil unless the report is this
+// account's and this app's: a report is the box's word about a route, and a
+// route stamped to another app is not this record's (instanceHealth's rule).
+func (j boxJoin) rollout(host, acct, app string) *rolloutView {
+ v, ok := j.rollouts[host]
+ if !ok || (!v.refused && (v.account != acct || v.app != app)) {
+ return nil
+ }
+ return &v
}
// dbSize is the join's database size for host, nil when no fresh box
@@ -379,8 +423,14 @@ func loadBoxJoin(ctx context.Context, s release.Store, now time.Time) boxJoin {
log.Printf("console: api instances: could not load status boxes for the list join: %v", err)
return boxJoin{}
}
+ return joinBoxes(boxes, now)
+}
+
+// joinBoxes is loadBoxJoin's fold over snapshots already read, so a page that
+// has its boxes reads them once.
+func joinBoxes(boxes []status.Box, now time.Time) boxJoin {
j := boxJoin{versions: map[string]string{}, upstreams: map[string][]status.UpstreamState{},
- dbBytes: map[string]int64{}}
+ dbBytes: map[string]int64{}, rollouts: map[string]rolloutView{}}
for _, b := range boxes {
if b.Metrics == nil || status.BoxStale(b, now, status.SnapshotPeriod) {
continue
@@ -394,6 +444,38 @@ func loadBoxJoin(ctx context.Context, s release.Store, now time.Time) boxJoin {
for host, n := range b.Metrics.RouteDBBytes {
j.dbBytes[host] = n
}
+ ro := b.Metrics.Rollout
+ if ro == nil {
+ continue
+ }
+ if ro.Unavailable != "" {
+ // The scheduler could not arm, so Hosts is empty. Say so for every
+ // host this box serves, from the version it reports for it.
+ for host, v := range b.Metrics.RouteVersions {
+ j.rollouts[host] = rolloutView{Box: b.Label, Installed: v, Unavailable: ro.Unavailable, refused: true}
+ }
+ continue
+ }
+ // Plan 5 marks every host a failure record holds as failed. The
+ // record's own Host is the one the build failed on; the rest are held
+ // by it (spec §9.1), so their attempt is cleared and HeldBy kept. A
+ // hold-mode host left down (AttemptDown) stays as it is.
+ failedOn := map[[4]string]bool{}
+ for _, f := range ro.Failures {
+ failedOn[[4]string{f.Account, f.App, f.Build, f.Host}] = true
+ }
+ for host, hr := range ro.Hosts {
+ if hr.Attempt == status.AttemptFailed && !failedOn[[4]string{hr.Account, hr.App, hr.AttemptBuild, host}] {
+ hr.Attempt, hr.AttemptBuild = "", ""
+ }
+ a := ro.Apps[hr.Account+"/"+hr.App]
+ j.rollouts[host] = rolloutView{
+ Box: b.Label, Installed: hr.Installed, Serving: hr.Serving, Good: hr.Good, Awake: hr.Awake,
+ Attempt: hr.Attempt, AttemptBuild: hr.AttemptBuild, HeldBy: hr.HeldBy, Reason: hr.Reason,
+ ConfigPending: hr.ConfigPending, SchedulerIdle: a.Queued == 0 && a.Running == 0 && a.Settling == 0,
+ account: hr.Account, app: hr.App,
+ }
+ }
}
return j
}
@@ -455,7 +537,8 @@ func (s *Server) handleAPIInstanceList(w http.ResponseWriter, r *http.Request) {
}
matched = append(matched, instanceListRecord{Record: rec,
StagedVersion: join.versions[rec.Host], Health: h, Upstreams: join.upstreams[rec.Host],
- ServedBy: served[rec.Host], Gated: rec.Gate != "", DBBytes: join.dbSize(rec.Host)})
+ ServedBy: served[rec.Host], Gated: rec.Gate != "", DBBytes: join.dbSize(rec.Host),
+ Rollout: join.rollout(rec.Host, acct.Sqid, app)})
}
total := len(matched)
// provision.ListRecords returns records host-sorted and nothing above
diff --git a/internal/console/rolloutjoin_test.go b/internal/console/rolloutjoin_test.go
new file mode 100644
index 000000000..4f5dea320
--- /dev/null
+++ b/internal/console/rolloutjoin_test.go
@@ -0,0 +1,173 @@
+package console
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+func TestAPIInstanceListCarriesTheRollout(t *testing.T) {
+ fx, token := instanceFixture(t, "code-inst-rollout")
+ enableInstances(t, fx, "titogo", titoManifest())
+ u := instancesURL(fx, "bab", "titogo")
+ for _, h := range []string{"acme.go.tito.io", "beta.go.tito.io"} {
+ if code, _, out := rawReq(t, http.MethodPost, u, token, fmt.Sprintf(`{"host":%q,"region":"ie"}`, h)); code != http.StatusCreated {
+ t.Fatalf("create %s = %d %s", h, code, out)
+ }
+ }
+ box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now())
+ box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{
+ Hosts: map[string]status.HostRollout{
+ "acme.go.tito.io": {Account: "bab", App: "titogo", Installed: "v2", Serving: "v1", Good: "v1",
+ Awake: true, Attempt: status.AttemptRunning, AttemptBuild: "v2"},
+ // A report naming another app under this host: never joined.
+ "beta.go.tito.io": {Account: "bab", App: "other", Installed: "v9", Serving: "v9"},
+ },
+ Apps: map[string]status.AppRollout{"bab/titogo": {Running: 1}},
+ }}
+ if err := status.WriteBox(context.Background(), fx.rel, box); err != nil {
+ t.Fatal(err)
+ }
+ code, _, out := rawReq(t, http.MethodGet, u, token, "")
+ if code != http.StatusOK {
+ t.Fatalf("list = %d %s", code, out)
+ }
+ var list struct {
+ Instances []struct {
+ Host string `json:"host"`
+ Rollout *rolloutView `json:"rollout"`
+ } `json:"instances"`
+ }
+ if err := json.Unmarshal([]byte(out), &list); err != nil {
+ t.Fatal(err)
+ }
+ got := map[string]*rolloutView{}
+ for _, r := range list.Instances {
+ got[r.Host] = r.Rollout
+ }
+ a := got["acme.go.tito.io"]
+ if a == nil || a.Box != box.Label || a.Installed != "v2" || a.Serving != "v1" || a.Attempt != status.AttemptRunning || a.SchedulerIdle {
+ t.Fatalf("acme rollout = %+v", a)
+ }
+ if got["beta.go.tito.io"] != nil {
+ t.Fatalf("beta joined another app's report: %+v", got["beta.go.tito.io"])
+ }
+}
+
+func TestJoinBoxesIdleNeedsNothingQueuedRunningOrSettling(t *testing.T) {
+ idle := func(a status.AppRollout) bool {
+ b := status.Box{GeneratedAt: time.Now(), Label: "x", Metrics: &status.BoxMetrics{Rollout: &status.RolloutReport{
+ Hosts: map[string]status.HostRollout{"h": {Account: "bab", App: "shop", Installed: "v1"}},
+ Apps: map[string]status.AppRollout{"bab/shop": a},
+ }}}
+ return joinBoxes([]status.Box{b}, time.Now()).rollout("h", "bab", "shop").SchedulerIdle
+ }
+ if !idle(status.AppRollout{}) {
+ t.Error("an empty scheduler is not idle")
+ }
+ for _, a := range []status.AppRollout{{Queued: 1}, {Running: 1}, {Settling: 1}} {
+ if idle(a) {
+ t.Errorf("%+v reads as idle", a)
+ }
+ }
+}
+
+// Plan 5 reports every host a failure record holds as failed. Spec §9.1:
+// the one the build failed on is failed, its siblings are held.
+func TestJoinBoxesTellsFailedFromHeld(t *testing.T) {
+ failed := status.HostRollout{Account: "bab", App: "shop", Installed: "v2", Serving: "v1", Good: "v1", Awake: true,
+ Attempt: status.AttemptFailed, AttemptBuild: "v2", HeldBy: "v2", Reason: status.ReasonHealth}
+ b := status.Box{GeneratedAt: time.Now(), Label: "x", Metrics: &status.BoxMetrics{Rollout: &status.RolloutReport{
+ Hosts: map[string]status.HostRollout{"a": failed, "b": failed},
+ Apps: map[string]status.AppRollout{"bab/shop": {}},
+ Failures: []status.RolloutFailure{{Account: "bab", App: "shop", Build: "v2", Host: "a", Reason: status.ReasonHealth}},
+ }}}
+ j := joinBoxes([]status.Box{b}, time.Now())
+ if a := j.rollout("a", "bab", "shop"); a.Attempt != status.AttemptFailed || a.AttemptBuild != "v2" || a.HeldBy != "v2" {
+ t.Errorf("a, where v2 failed = %+v", a)
+ }
+ if sib := j.rollout("b", "bab", "shop"); sib.Attempt != "" || sib.AttemptBuild != "" || sib.HeldBy != "v2" || sib.Reason != status.ReasonHealth {
+ t.Errorf("b, held by a's failure = %+v", sib)
+ }
+}
+
+func TestAPIPromoteHeldSaysWhenItLifts(t *testing.T) {
+ fx := newConsoleFixture(t)
+ if _, err := fx.st.CreateAppIn(OpsAccountID, "titogo", "paul@keymail.dev", time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ shipVersion(t, fx, "titogo", "abc1234")
+ seedChannel(t, fx, "titogo", "edge", "abc1234")
+ if err := release.SetProduction(context.Background(), fx.rel, fx.priv, release.OpsAccount, "titogo", true, "test"); err != nil {
+ t.Fatal(err)
+ }
+ token := cliToken(t, fx, "paul@keymail.dev", "code-held-until")
+ before := time.Now()
+ code, body := postRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/promote",
+ map[string]string{"version": "abc1234", "channel": "beta"}, token)
+ if code != http.StatusOK || !held(t, body) {
+ t.Fatalf("beta promote = %d %s", code, body)
+ }
+ var out struct {
+ HeldUntil string `json:"held_until"`
+ }
+ if err := json.Unmarshal([]byte(body), &out); err != nil {
+ t.Fatal(err)
+ }
+ until, err := time.Parse(time.RFC3339, out.HeldUntil)
+ if err != nil {
+ t.Fatalf("held_until = %q: %v", out.HeldUntil, err)
+ }
+ window := release.HoldFor(release.DefaultHolds, "beta")
+ if until.Before(before.Add(window).Add(-time.Minute)) || until.After(time.Now().Add(window).Add(time.Minute)) {
+ t.Fatalf("held_until = %s, want about now + %s", until, window)
+ }
+ code, body = postRaw(t, fx.ts.URL+"/api/cli/apps/bab/titogo/promote",
+ map[string]string{"version": "abc1234", "channel": "canary/x"}, token)
+ var canary struct {
+ HeldUntil string `json:"held_until"`
+ }
+ if code != http.StatusOK || held(t, body) || json.Unmarshal([]byte(body), &canary) != nil || canary.HeldUntil != "" {
+ t.Fatalf("canary promote = %d %s", code, body)
+ }
+}
+
+// A box that armed nothing says why (CARLOS_SWAP is on but the scheduler could
+// not arm): its Hosts map is empty, so the join answers from the box's route
+// versions with the reason code and nothing else.
+func TestJoinBoxesCarriesTheRefusalOfAnUnavailableBox(t *testing.T) {
+ b := status.Box{GeneratedAt: time.Now(), Label: "refusing", Metrics: &status.BoxMetrics{
+ RouteVersions: map[string]string{"h": "v3"},
+ Rollout: &status.RolloutReport{Unavailable: "no-deploy-store"},
+ }}
+ got := joinBoxes([]status.Box{b}, time.Now()).rollout("h", "bab", "shop")
+ if got == nil || got.Box != "refusing" || got.Installed != "v3" || got.Unavailable != "no-deploy-store" ||
+ got.Serving != "" || got.Attempt != "" || got.SchedulerIdle {
+ t.Fatalf("refusing box rollout = %+v", got)
+ }
+ if joinBoxes([]status.Box{b}, time.Now()).rollout("other", "bab", "shop") != nil {
+ t.Error("a host the refusing box does not list joined a rollout")
+ }
+ raw, err := json.Marshal(got)
+ if err != nil || !strings.Contains(string(raw), `"unavailable":"no-deploy-store"`) {
+ t.Fatalf("wire = %s, %v", raw, err)
+ }
+}
+
+// A box with no rollout report (CARLOS_SWAP off, or from before plan 5) joins
+// nothing: the flag-off wire is unchanged.
+func TestJoinBoxesNilReportJoinsNothing(t *testing.T) {
+ b := status.Box{GeneratedAt: time.Now(), Label: "old", Metrics: &status.BoxMetrics{
+ RouteVersions: map[string]string{"h": "v3"},
+ }}
+ if got := joinBoxes([]status.Box{b}, time.Now()).rollout("h", "bab", "shop"); got != nil {
+ t.Fatalf("nil report joined %+v", got)
+ }
+}
diff --git a/internal/provision/plan9_b8_census_test.go b/internal/provision/plan9_b8_census_test.go
index 031e355ed..5237a17b4 100644
--- a/internal/provision/plan9_b8_census_test.go
+++ b/internal/provision/plan9_b8_census_test.go
@@ -560,7 +560,7 @@ var plan9B8CensusInventory = []censusEntry{
{"internal/console/apiinstancemove.go", "Server.handleAPIInstanceMoveRequest", "record-writer", "key:SetRecordMove,lit:instances", "708ef4614cf5fd9c1ad382cb79da55af0b1192a235fd719765affda92aaf1668"},
{"internal/console/apiinstancemove.go", "Server.registerMovesPhase1Routes", "key-site", "lit:instances", "5af589d288ba22856f631481c40cc001aac342ea74cee6ee0464ec77d0320bbc"},
{"internal/console/apiinstances.go", "Server.instanceManifest", "key-site", "lit:instances", "e6160ffe84abda9163527ccb33e2ffff2109b35908adb2f3a754534325658fdf"},
- {"internal/console/apiinstances.go", "Server.handleAPIInstanceList", "key-site", "lit:instances", "54da4ed10eb3b8cce0dd64407221befa232f135b81b9c76840aa42bcd5b93106"},
+ {"internal/console/apiinstances.go", "Server.handleAPIInstanceList", "key-site", "lit:instances", "429011bf71564e430cdbe92fbb347d0d200c52c2cb392aeb0864a1f75bac18a1"},
{"internal/console/apiinstances.go", "Server.handleAPIInstanceCreate", "record-writer", "key:CreateRecord", "20f862446f119f25d7d72eab6edd1b7cfb1528afd5b4c352e0cb8442f5080b13"},
{"internal/console/apiinstances.go", "Server.handleAPIInstanceUpdate", "record-writer", "key:UpdateRecord", "4b792c410399631eb3b38c78829df85dda045651bbc32df769767cf52f015b6d"},
{"internal/console/apiscan.go", "Server.writeScanRecord", "nonrecord", "sel:Put", "c3c092f4b7be9bf6bf09d0fe1e2c9fe52892ee97e57f6c7ab964b85fe36fb5c6"},