amadan

carlos / platform Public

zdd-6 final fix wave 1: W3 one fleet-aware rollout join for the list, the page and the retry door

loadRolloutJoin reads the platform's boxes and the rollout reports of this account's own customer-fleet boxes (fleets/<name>/), keeps boxJoin.rollout's account/app check, lets a platform box win over a fleet box for the same host, and feeds rollout_incomplete. Another account's fleet is never read. Astra plan 6 R1 #6; reverses the L26 box-scope ruling.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude cb24f42e3294df5e512311402c55a4c7aa5bdaec parent 1bc19af
5 files changed, +269 −37
  • internal/console/apiinstances.go +114 −30
  • internal/console/apiretry.go +4 −2
  • internal/console/instancepage.go +5 −3
  • internal/console/rolloutfleet_test.go +144 −0
  • internal/provision/plan9_b8_census_test.go +2 −2
diff --git a/internal/console/apiinstances.go b/internal/console/apiinstances.go
index c27307fe6..7da7f155b 100644
--- a/internal/console/apiinstances.go
+++ b/internal/console/apiinstances.go
@@ -438,6 +438,60 @@ func loadBoxJoinComplete(ctx context.Context, s release.Store, now time.Time) (b
return joinBoxes(boxes, now), complete
}
+// loadRolloutJoin is the box join the instance list, the instance page and
+// the host-retry door share (Astra plan 6 R1 #6), so the three never
+// disagree about a host's rollout: the platform's boxes (status/boxes/),
+// then the rollout reports of the customer-fleet boxes of THIS account's
+// fleets (fleets/<name>/status/boxes/, the prefix systemBoxes reads). A
+// fleet another account owns is never read: fleet names are one flat
+// namespace across accounts, and its boxes are written by machines this
+// account does not control. boxJoin.rollout still checks the report's own
+// account and app.
+//
+// Hosts are assumed unique to one box. When two boxes name the same host (a
+// move in flight, a report not yet stale), the last platform box read wins,
+// as loadBoxJoin always did, and a platform box wins over any fleet box,
+// then the first fleet box read wins: the operator's boxes are the ones this
+// console trusts most.
+//
+// complete is false when any snapshot, fleet listing or registry read
+// failed, so the list can say rollout_incomplete. A console with no route
+// registry, or none yet, has no fleets and is complete on its platform
+// boxes alone.
+func (s *Server) loadRolloutJoin(ctx context.Context, acct string, now time.Time) (boxJoin, bool) {
+ j, complete := loadBoxJoinComplete(ctx, s.cfg.Releases, now)
+ if s.cfg.Releases == nil || s.cfg.Registry == "" {
+ return j, complete
+ }
+ reg, err := s.openPassRegistry()
+ switch {
+ case errors.Is(err, os.ErrNotExist):
+ return j, complete // no registry file yet: no fleets
+ case err != nil:
+ log.Printf("console: rollout join %s: open registry: %v", acct, err)
+ return j, false
+ }
+ fleets, err := reg.ListFleets()
+ if err != nil {
+ log.Printf("console: rollout join %s: list fleets: %v", acct, err)
+ return j, false
+ }
+ for _, f := range fleets {
+ if f.Account != acct {
+ continue
+ }
+ boxes, fleetComplete, err := status.LoadBoxesComplete(ctx, release.NewPrefixStore(s.cfg.Releases, "fleets/"+f.Name+"/"))
+ if err != nil {
+ log.Printf("console: rollout join %s: fleet %s boxes: %v", acct, f.Name, err)
+ complete = false
+ continue
+ }
+ complete = complete && fleetComplete
+ j.joinFleetRollouts(boxes, now)
+ }
+ return j, complete
+}
+
// joinBoxes is loadBoxJoin's fold over snapshots already read, so a page that
// has its boxes reads them once.
func joinBoxes(boxes []status.Box, now time.Time) boxJoin {
@@ -459,40 +513,70 @@ func joinBoxes(boxes []status.Box, now time.Time) boxJoin {
for host, n := range b.Metrics.RouteDBBytes {
j.dbBytes[host] = n
}
- ro := b.Metrics.Rollout
- if ro == nil {
- continue
+ j.foldRollout(b, true)
+ }
+ return j
+}
+
+// foldRollout adds box b's rollout report to the join. override false keeps
+// a host the join already has: the caller folds platform boxes first, and a
+// fleet box's word about a host never replaces a platform box's.
+func (j *boxJoin) foldRollout(b status.Box, override bool) {
+ ro := b.Metrics.Rollout
+ if ro == nil {
+ return
+ }
+ if j.rollouts == nil {
+ j.rollouts = map[string]rolloutView{}
+ }
+ put := func(host string, v rolloutView) {
+ if _, had := j.rollouts[host]; had && !override {
+ return
}
- if ro.Unavailable != "" {
- // The scheduler could not arm, so Hosts is empty. Say so for every
- // host this box serves, from the version it reports for it.
- for host, v := range b.Metrics.RouteVersions {
- j.rollouts[host] = rolloutView{Box: b.Label, Installed: v, Unavailable: ro.Unavailable, refused: true}
- }
- continue
+ j.rollouts[host] = v
+ }
+ if ro.Unavailable != "" {
+ // The scheduler could not arm, so Hosts is empty. Say so for every
+ // host this box serves, from the version it reports for it.
+ for host, v := range b.Metrics.RouteVersions {
+ put(host, rolloutView{Box: b.Label, Installed: v, Unavailable: ro.Unavailable, refused: true})
}
- // Plan 5 marks every host a failure record holds as failed. The
- // record's own Host is the one the build failed on; the rest are held
- // by it (spec §9.1), so their attempt is cleared and HeldBy kept. A
- // hold-mode host left down (AttemptDown) stays as it is.
- failedOn := map[[4]string]bool{}
- for _, f := range ro.Failures {
- failedOn[[4]string{f.Account, f.App, f.Build, f.Host}] = true
+ return
+ }
+ // Plan 5 marks every host a failure record holds as failed. The
+ // record's own Host is the one the build failed on; the rest are held
+ // by it (spec §9.1), so their attempt is cleared and HeldBy kept. A
+ // hold-mode host left down (AttemptDown) stays as it is.
+ failedOn := map[[4]string]bool{}
+ for _, f := range ro.Failures {
+ failedOn[[4]string{f.Account, f.App, f.Build, f.Host}] = true
+ }
+ for host, hr := range ro.Hosts {
+ if hr.Attempt == status.AttemptFailed && !failedOn[[4]string{hr.Account, hr.App, hr.AttemptBuild, host}] {
+ hr.Attempt, hr.AttemptBuild = "", ""
}
- for host, hr := range ro.Hosts {
- if hr.Attempt == status.AttemptFailed && !failedOn[[4]string{hr.Account, hr.App, hr.AttemptBuild, host}] {
- hr.Attempt, hr.AttemptBuild = "", ""
- }
- a := ro.Apps[hr.Account+"/"+hr.App]
- j.rollouts[host] = rolloutView{
- Box: b.Label, Installed: hr.Installed, Serving: hr.Serving, Good: hr.Good, Awake: hr.Awake,
- Attempt: hr.Attempt, AttemptBuild: hr.AttemptBuild, HeldBy: hr.HeldBy, Reason: hr.Reason,
- ConfigPending: hr.ConfigPending, SchedulerIdle: a.Queued == 0 && a.Running == 0 && a.Settling == 0,
- account: hr.Account, app: hr.App,
- }
+ a := ro.Apps[hr.Account+"/"+hr.App]
+ put(host, rolloutView{
+ Box: b.Label, Installed: hr.Installed, Serving: hr.Serving, Good: hr.Good, Awake: hr.Awake,
+ Attempt: hr.Attempt, AttemptBuild: hr.AttemptBuild, HeldBy: hr.HeldBy, Reason: hr.Reason,
+ ConfigPending: hr.ConfigPending, SchedulerIdle: a.Queued == 0 && a.Running == 0 && a.Settling == 0,
+ account: hr.Account, app: hr.App,
+ })
+ }
+}
+
+// joinFleetRollouts folds customer-fleet boxes into the join: their rollout
+// reports only, and never over a host a platform box already reported. Their
+// route versions, upstreams and database sizes stay out, so the list's other
+// columns are the platform join's, as before. Staleness is the platform's
+// rule (status.BoxStale).
+func (j *boxJoin) joinFleetRollouts(boxes []status.Box, now time.Time) {
+ for _, b := range boxes {
+ if b.Metrics == nil || status.BoxStale(b, now, status.SnapshotPeriod) {
+ continue
}
+ j.foldRollout(b, false)
}
- return j
}
// handleAPIInstanceList answers `carlos instances list`: this app's records,
@@ -521,7 +605,7 @@ func (s *Server) handleAPIInstanceList(w http.ResponseWriter, r *http.Request) {
jsonError(w, http.StatusBadGateway, "could not list instances")
return
}
- join, joinComplete := loadBoxJoinComplete(ctx, s.cfg.Releases, time.Now())
+ join, joinComplete := s.loadRolloutJoin(ctx, acct.Sqid, time.Now())
served := loadEdgesJoin(ctx, s.cfg.Releases)
params := r.URL.Query()
q := parseInstanceQuery(params)
diff --git a/internal/console/apiretry.go b/internal/console/apiretry.go
index 343dff56f..273adbcaf 100644
--- a/internal/console/apiretry.go
+++ b/internal/console/apiretry.go
@@ -38,7 +38,8 @@ var (
// retryHostBuild is the build holding host back, as its box reports it. The
// record must exist in this account and app (the instance family's 404), and
-// the report must be this app's own (boxJoin.rollout). A box that could not
+// the report must be this app's own (boxJoin.rollout), read through the same
+// loader as the list and the instance page (loadRolloutJoin). A box that could not
// arm its scheduler reports no hold, so its host is not held.
func (s *Server) retryHostBuild(ctx context.Context, acct *Account, app, host string) (string, error) {
if _, err := provision.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app, host); err != nil {
@@ -47,7 +48,8 @@ func (s *Server) retryHostBuild(ctx context.Context, acct *Account, app, host st
}
return "", err
}
- ro := loadBoxJoin(ctx, s.cfg.Releases, time.Now()).rollout(host, acct.Sqid, app)
+ join, _ := s.loadRolloutJoin(ctx, acct.Sqid, time.Now())
+ ro := join.rollout(host, acct.Sqid, app)
if ro == nil || ro.Unavailable != "" || ro.HeldBy == "" {
return "", errRetryNotHeld
}
diff --git a/internal/console/instancepage.go b/internal/console/instancepage.go
index 412f7473d..586ca9f2c 100644
--- a/internal/console/instancepage.go
+++ b/internal/console/instancepage.go
@@ -233,9 +233,11 @@ func (s *Server) handleAppInstance(w http.ResponseWriter, r *http.Request) {
if boxes, _, err := s.systemBoxes(r.Context(), false); err == nil {
detail.Layout = layoutWordFor(boxes, host)
}
- // The rollout view comes from the same platform-box join the retry
- // door reads (loadBoxJoin), not from systemBoxes.
- detail.Rollout = loadBoxJoin(r.Context(), s.cfg.Releases, time.Now()).rollout(host, acct.Sqid, p.AppName)
+ // The rollout view comes from the join the list and the retry door
+ // read (loadRolloutJoin: platform boxes and this account's fleets),
+ // not from systemBoxes, which reads every account's fleets.
+ join, _ := s.loadRolloutJoin(r.Context(), acct.Sqid, time.Now())
+ detail.Rollout = join.rollout(host, acct.Sqid, p.AppName)
}
if reg, err := openRegistry(s.cfg.Registry); err == nil {
if aliases, err := reg.ListAliases(); err == nil {
diff --git a/internal/console/rolloutfleet_test.go b/internal/console/rolloutfleet_test.go
new file mode 100644
index 000000000..2791dcbe8
--- /dev/null
+++ b/internal/console/rolloutfleet_test.go
@@ -0,0 +1,144 @@
+package console
+
+import (
+ "context"
+ "encoding/json"
+ "fmt"
+ "net/http"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/rolloutwords"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+// writeFleetHeldBox registers fleet to owner and writes, under the fleet's
+// own prefix, a box whose report holds host back on build for this
+// fixture's account and app. It returns the box's label.
+func writeFleetHeldBox(t *testing.T, fx *consoleFixture, fleet, owner, host, app, build string) string {
+ t.Helper()
+ reg, err := registry.Open(fx.regPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer reg.Close()
+ if err := reg.UpsertFleet(registry.Fleet{Name: fleet, Account: owner, Bucket: fleet + "-bucket"}); err != nil {
+ t.Fatal(err)
+ }
+ box := status.CollectBox(nil, status.Probes{MachineID: fleet + "-box"}, time.Now())
+ box.Metrics = &status.BoxMetrics{Rollout: &status.RolloutReport{Hosts: map[string]status.HostRollout{
+ host: {Account: fx.sqid, App: app, Installed: build, Serving: "v1", Good: "v1", Awake: true, HeldBy: build, Reason: status.ReasonHealth},
+ }}}
+ if err := status.WriteBox(context.Background(), release.NewPrefixStore(fx.rel, "fleets/"+fleet+"/"), box); err != nil {
+ t.Fatal(err)
+ }
+ return box.Label
+}
+
+// listRollout is the list's answer for host: its rollout view, and whether
+// the answer said rollout_incomplete.
+func listRollout(t *testing.T, fx *consoleFixture, token, app, host string) (*rolloutView, bool) {
+ t.Helper()
+ code, _, out := rawReq(t, http.MethodGet, fx.ts.URL+"/api/cli/apps/"+fx.sqid+"/"+app+"/instances", token, "")
+ if code != http.StatusOK {
+ t.Fatalf("list = %d %s", code, out)
+ }
+ var list struct {
+ Instances []struct {
+ Host string `json:"host"`
+ Rollout *rolloutView `json:"rollout"`
+ } `json:"instances"`
+ RolloutIncomplete bool `json:"rollout_incomplete"`
+ }
+ if err := json.Unmarshal([]byte(out), &list); err != nil {
+ t.Fatal(err)
+ }
+ for _, r := range list.Instances {
+ if r.Host == host {
+ return r.Rollout, list.RolloutIncomplete
+ }
+ }
+ t.Fatalf("list lacks %s: %s", host, out)
+ return nil, false
+}
+
+// Astra plan 6 R1 #6: a customer-fleet box's report counts. Its held host
+// shows held on the list and on the instance page, and the retry door takes
+// the retry, through the one loader all three share.
+func TestFleetBoxHoldShowsOnTheListPageAndRetryDoor(t *testing.T) {
+ fx := newConsoleFixture(t)
+ fx.addRoute(t, registry.Route{Host: heldHost, App: "kass", Account: fx.sqid, Kind: "instance", Status: "ready", Version: "v2", Channel: "stable"})
+ sibling(t, fx, "kass", heldHost, "stable")
+ label := writeFleetHeldBox(t, fx, "pi", fx.sqid, heldHost, "kass", "v2")
+ token := cliToken(t, fx, "paul@keymail.dev", "code-fleet-held")
+
+ ro, incomplete := listRollout(t, fx, token, "kass", heldHost)
+ if ro == nil || ro.Box != label || ro.HeldBy != "v2" || incomplete {
+ t.Fatalf("list rollout = %+v, incomplete %v: want the fleet box's hold, complete", ro, incomplete)
+ }
+ c := signInPaul(t, fx)
+ if code, _, body := get(t, c, fx.u("/apps/kass/instances/"+heldHost)); code != http.StatusOK ||
+ !strings.Contains(body, fmt.Sprintf(swapCopy.HoldText, "v2", rolloutwords.Reason(status.ReasonHealth), "v1")) {
+ t.Fatalf("page = %d, lacks the fleet box's hold", code)
+ }
+ code, _, body := rawReq(t, http.MethodPost, fx.ts.URL+"/api/cli/apps/"+fx.sqid+"/kass/instances/"+heldHost+"/retry", token, `{}`)
+ if code != http.StatusOK || !strings.Contains(body, `"version":"v2"`) {
+ t.Fatalf("retry = %d %s, want 200 for v2", code, body)
+ }
+
+ // A fleet box snapshot that cannot be read makes the join incomplete.
+ if _, err := fx.rel.Put(context.Background(), "fleets/pi/status/boxes/broken.json", []byte("{not json"), ""); err != nil {
+ t.Fatal(err)
+ }
+ if _, incomplete := listRollout(t, fx, token, "kass", heldHost); !incomplete {
+ t.Fatal("an unreadable fleet box snapshot left the join complete")
+ }
+}
+
+// Astra plan 6 R1 #6: only this account's fleets are read. Another
+// account's fleet box that names this account's host is never joined,
+// whatever account its report claims: no hold on the list or the page, and
+// the retry door refuses.
+func TestAnotherAccountsFleetBoxIsNeverJoined(t *testing.T) {
+ fx := newConsoleFixture(t)
+ fx.addRoute(t, registry.Route{Host: heldHost, App: "kass", Account: fx.sqid, Kind: "instance", Status: "ready", Version: "v2", Channel: "stable"})
+ sibling(t, fx, "kass", heldHost, "stable")
+ writeFleetHeldBox(t, fx, "stranger", "zzz", heldHost, "kass", "v2")
+ token := cliToken(t, fx, "paul@keymail.dev", "code-fleet-stranger")
+
+ if ro, _ := listRollout(t, fx, token, "kass", heldHost); ro != nil {
+ t.Fatalf("list joined another account's fleet box: %+v", ro)
+ }
+ c := signInPaul(t, fx)
+ if _, _, body := get(t, c, fx.u("/apps/kass/instances/"+heldHost)); strings.Contains(body, swapCopy.Installed) {
+ t.Fatal("page joined another account's fleet box")
+ }
+ code, _, body := rawReq(t, http.MethodPost, fx.ts.URL+"/api/cli/apps/"+fx.sqid+"/kass/instances/"+heldHost+"/retry", token, `{}`)
+ if code != http.StatusConflict || !strings.Contains(body, fmt.Sprintf(apiRetryNotHeld, heldHost)) {
+ t.Fatalf("retry = %d %s, want 409", code, body)
+ }
+}
+
+// A platform box's report wins over a fleet box's for the same host: the
+// operator's boxes are the ones the console trusts most.
+func TestJoinFleetRolloutsNeverOverrideAPlatformBox(t *testing.T) {
+ now := time.Now()
+ view := func(label, build string) status.Box {
+ return status.Box{GeneratedAt: now, Label: label, Metrics: &status.BoxMetrics{Rollout: &status.RolloutReport{
+ Hosts: map[string]status.HostRollout{"h": {Account: "bab", App: "shop", Installed: build}},
+ }}}
+ }
+ j := joinBoxes([]status.Box{view("platform", "v1")}, now)
+ j.joinFleetRollouts([]status.Box{view("fleet", "v9")}, now)
+ if got := j.rollout("h", "bab", "shop"); got == nil || got.Box != "platform" || got.Installed != "v1" {
+ t.Fatalf("h = %+v, want the platform box's report", got)
+ }
+ var empty boxJoin
+ empty.joinFleetRollouts([]status.Box{view("fleet", "v9")}, now)
+ if got := empty.rollout("h", "bab", "shop"); got == nil || got.Box != "fleet" {
+ t.Fatalf("h on an empty join = %+v, want the fleet box's report", got)
+ }
+}
diff --git a/internal/provision/plan9_b8_census_test.go b/internal/provision/plan9_b8_census_test.go
index ead7ab3b0..7d40cceee 100644
--- a/internal/provision/plan9_b8_census_test.go
+++ b/internal/provision/plan9_b8_census_test.go
@@ -584,7 +584,7 @@ var plan9B8CensusInventory = []censusEntry{
{"internal/console/apiinstancemove.go", "Server.handleAPIInstanceMoveRequest", "record-writer", "key:SetRecordMove,lit:instances", "708ef4614cf5fd9c1ad382cb79da55af0b1192a235fd719765affda92aaf1668"},
{"internal/console/apiinstancemove.go", "Server.registerMovesPhase1Routes", "key-site", "lit:instances", "5af589d288ba22856f631481c40cc001aac342ea74cee6ee0464ec77d0320bbc"},
{"internal/console/apiinstances.go", "Server.instanceManifest", "key-site", "lit:instances", "e6160ffe84abda9163527ccb33e2ffff2109b35908adb2f3a754534325658fdf"},
- {"internal/console/apiinstances.go", "Server.handleAPIInstanceList", "key-site", "lit:instances", "e6b371f7911ee013f1074b983f7d582b5b07982784b375e18304e877a5d4dd39"},
+ {"internal/console/apiinstances.go", "Server.handleAPIInstanceList", "key-site", "lit:instances", "e1ef38cb728c50625690ce16ffd49c4548d6cb2e5e5277197611cf7cbfd6df47"},
{"internal/console/apiinstances.go", "Server.handleAPIInstanceCreate", "record-writer", "key:CreateRecord", "20f862446f119f25d7d72eab6edd1b7cfb1528afd5b4c352e0cb8442f5080b13"},
{"internal/console/apiinstances.go", "Server.handleAPIInstanceUpdate", "record-writer", "key:UpdateRecord", "4b792c410399631eb3b38c78829df85dda045651bbc32df769767cf52f015b6d"},
{"internal/console/apiscan.go", "Server.writeScanRecord", "nonrecord", "sel:Put", "c3c092f4b7be9bf6bf09d0fe1e2c9fe52892ee97e57f6c7ab964b85fe36fb5c6"},
@@ -617,7 +617,7 @@ var plan9B8CensusInventory = []censusEntry{
{"internal/console/geoippublish.go", "geoipPublisher.ExpireIfDue", "nonrecord", "sel:Put", "a7d4f24a25c7bda40f18268d34a99203b58aa07bc10eda0db2306b54cb08302b"},
{"internal/console/hostreports.go", "putHostReport", "nonrecord", "sel:Put", "d8ba8c77b5f0ee5bcde5c11f4a608cf2bd2491c36b8cb9bf26753a230d485ad4"},
{"internal/console/infrapass.go", "Server.putInventory", "nonrecord", "sel:Put", "c5d930ec178ee206d25194a2b265738ac42c6f00bd9b73626e4d46d769d370ad"},
- {"internal/console/instancepage.go", "Server.handleAppInstance", "key-site", "lit:instances", "95147b261febf9a40976bef10c6a55b8c328ef0412745d991b3372e2a7d3be10"},
+ {"internal/console/instancepage.go", "Server.handleAppInstance", "key-site", "lit:instances", "29e4e5717910dd5ea943ad3a0bef7ec2ef835b83e8d0858c9a29c4ea5940721a"},
{"internal/console/relayops.go", "Server.relayAcquire", "nonrecord", "sel:Put", "3c757e1c92f1c0ce41c3b5f00aa1397ce5a0c4faabc2f176b23355cf5a41f5fb"},
{"internal/console/relayops.go", "Server.relayPublishOnce", "nonrecord", "sel:Put", "783a3c20337914e08f92be783f6b48ab14288437a8043c0d5bf29f281a7a9b1d"},
{"internal/console/relayops.go", "Server.relayRotate", "nonrecord", "sel:Put", "58e0090a4990e2d5e793cec2ec7bcbfb3626b4043307f64484a55766099e1434"},