amadan
zdd-6 final fix wave 2: X2 rollout_incomplete counts only sources that could hold the app's reports
The list's completeness covers platform boxes and only the account's fleets one of the app's records is placed on (Region fleet/<name>, or either end of a move). An unreadable snapshot on an unrelated fleet no longer blocks a healthy app's legacy watch. The join's content, the page and the retry door are unchanged. Astra plan 6 R2 new Medium.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
5 files changed,
+108
−17
internal/console/apiinstances.go+38 −10internal/console/apiretry.go+1 −1internal/console/instancepage.go+1 −1internal/console/rolloutfleet_test.go+66 −3internal/provision/plan9_b8_census_test.go+2 −2
diff --git a/internal/console/apiinstances.go b/internal/console/apiinstances.go| index 7da7f155b..294643731 100644 |
| --- a/internal/console/apiinstances.go |
| +++ b/internal/console/apiinstances.go |
| @@ -454,27 +454,35 @@ func loadBoxJoinComplete(ctx context.Context, s release.Store, now time.Time) (b |
| // then the first fleet box read wins: the operator's boxes are the ones this |
| // console trusts most. |
| // |
| -// complete is false when any snapshot, fleet listing or registry read |
| -// failed, so the list can say rollout_incomplete. A console with no route |
| -// registry, or none yet, has no fleets and is complete on its platform |
| -// boxes alone. |
| -func (s *Server) loadRolloutJoin(ctx context.Context, acct string, now time.Time) (boxJoin, bool) { |
| +// complete is false when a read failed on a source that could hold the |
| +// reports in question: any platform box snapshot, and the fleets relevant |
| +// names (Astra plan 6 R2: an unreadable snapshot on a fleet none of the |
| +// app's instances is placed on says nothing about the app, and must not |
| +// block its deploy watch's fallback). relevant nil counts every fleet of the |
| +// account. A registry that cannot be read counts only when relevant names |
| +// some fleet. A console with no route registry, or none yet, has no fleets |
| +// and is complete on its platform boxes alone. Every fleet of the account is |
| +// still READ, whatever relevant says: completeness is scoped, the join's |
| +// content is not. |
| +func (s *Server) loadRolloutJoin(ctx context.Context, acct string, now time.Time, relevant map[string]bool) (boxJoin, bool) { |
| j, complete := loadBoxJoinComplete(ctx, s.cfg.Releases, now) |
| if s.cfg.Releases == nil || s.cfg.Registry == "" { |
| return j, complete |
| } |
| + counts := func(fleet string) bool { return relevant == nil || relevant[fleet] } |
| + unreadable := relevant == nil || len(relevant) > 0 |
| reg, err := s.openPassRegistry() |
| switch { |
| case errors.Is(err, os.ErrNotExist): |
| return j, complete // no registry file yet: no fleets |
| case err != nil: |
| log.Printf("console: rollout join %s: open registry: %v", acct, err) |
| - return j, false |
| + return j, complete && !unreadable |
| } |
| fleets, err := reg.ListFleets() |
| if err != nil { |
| log.Printf("console: rollout join %s: list fleets: %v", acct, err) |
| - return j, false |
| + return j, complete && !unreadable |
| } |
| for _, f := range fleets { |
| if f.Account != acct { |
| @@ -483,15 +491,35 @@ func (s *Server) loadRolloutJoin(ctx context.Context, acct string, now time.Time |
| boxes, fleetComplete, err := status.LoadBoxesComplete(ctx, release.NewPrefixStore(s.cfg.Releases, "fleets/"+f.Name+"/")) |
| if err != nil { |
| log.Printf("console: rollout join %s: fleet %s boxes: %v", acct, f.Name, err) |
| - complete = false |
| + complete = complete && !counts(f.Name) |
| continue |
| } |
| - complete = complete && fleetComplete |
| + complete = complete && (fleetComplete || !counts(f.Name)) |
| j.joinFleetRollouts(boxes, now) |
| } |
| return j, complete |
| } |
| +// recordFleets is the set of customer fleets that could hold reports for |
| +// recs: the fleet each record is placed on (its Region, "fleet/<name>"), |
| +// and either end of a move in flight. Never nil, so an app with no |
| +// fleet-placed record counts no fleet at all. |
| +func recordFleets(recs []provision.Record) map[string]bool { |
| + out := map[string]bool{} |
| + for _, rec := range recs { |
| + places := []string{rec.Region} |
| + if rec.Move != nil { |
| + places = append(places, rec.Move.From, rec.Move.To) |
| + } |
| + for _, p := range places { |
| + if f, ok := registry.FleetPlacement(p); ok { |
| + out[f] = true |
| + } |
| + } |
| + } |
| + return out |
| +} |
| + |
| // joinBoxes is loadBoxJoin's fold over snapshots already read, so a page that |
| // has its boxes reads them once. |
| func joinBoxes(boxes []status.Box, now time.Time) boxJoin { |
| @@ -605,7 +633,7 @@ func (s *Server) handleAPIInstanceList(w http.ResponseWriter, r *http.Request) { |
| jsonError(w, http.StatusBadGateway, "could not list instances") |
| return |
| } |
| - join, joinComplete := s.loadRolloutJoin(ctx, acct.Sqid, time.Now()) |
| + join, joinComplete := s.loadRolloutJoin(ctx, acct.Sqid, time.Now(), recordFleets(recs)) |
| served := loadEdgesJoin(ctx, s.cfg.Releases) |
| params := r.URL.Query() |
| q := parseInstanceQuery(params) |
diff --git a/internal/console/apiretry.go b/internal/console/apiretry.go| index 273adbcaf..3977bc4da 100644 |
| --- a/internal/console/apiretry.go |
| +++ b/internal/console/apiretry.go |
| @@ -48,7 +48,7 @@ func (s *Server) retryHostBuild(ctx context.Context, acct *Account, app, host st |
| } |
| return "", err |
| } |
| - join, _ := s.loadRolloutJoin(ctx, acct.Sqid, time.Now()) |
| + join, _ := s.loadRolloutJoin(ctx, acct.Sqid, time.Now(), nil) |
| ro := join.rollout(host, acct.Sqid, app) |
| if ro == nil || ro.Unavailable != "" || ro.HeldBy == "" { |
| return "", errRetryNotHeld |
diff --git a/internal/console/instancepage.go b/internal/console/instancepage.go| index 4dcd1d9a6..387b6bd9d 100644 |
| --- a/internal/console/instancepage.go |
| +++ b/internal/console/instancepage.go |
| @@ -241,7 +241,7 @@ func (s *Server) handleAppInstance(w http.ResponseWriter, r *http.Request) { |
| // The rollout view comes from the join the list and the retry door |
| // read (loadRolloutJoin: platform boxes and this account's fleets), |
| // not from systemBoxes, which reads every account's fleets. |
| - join, _ := s.loadRolloutJoin(r.Context(), acct.Sqid, time.Now()) |
| + join, _ := s.loadRolloutJoin(r.Context(), acct.Sqid, time.Now(), nil) |
| detail.Rollout = join.rollout(host, acct.Sqid, p.AppName) |
| } |
| if reg, err := openRegistry(s.cfg.Registry); err == nil { |
diff --git a/internal/console/rolloutfleet_test.go b/internal/console/rolloutfleet_test.go| index 2791dcbe8..ccc08bc5d 100644 |
| --- a/internal/console/rolloutfleet_test.go |
| +++ b/internal/console/rolloutfleet_test.go |
| @@ -9,6 +9,7 @@ import ( |
| "testing" |
| "time" |
| + "github.com/carlosframework/platform/internal/provision" |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| "github.com/carlosframework/platform/internal/rolloutwords" |
| @@ -65,13 +66,27 @@ func listRollout(t *testing.T, fx *consoleFixture, token, app, host string) (*ro |
| return nil, false |
| } |
| +// placedRecord writes app's instance record for host, placed at region |
| +// ("ie", or "fleet/<name>" for a customer fleet). |
| +func placedRecord(t *testing.T, fx *consoleFixture, app, host, region string) { |
| + t.Helper() |
| + if err := provision.CreateRecord(context.Background(), fx.rel, provision.Record{ |
| + Schema: 1, Host: host, Slug: strings.SplitN(host, ".", 2)[0], |
| + App: app, Account: release.OpsAccount, Region: region, Channel: "stable", |
| + CreatedAt: time.Now().UTC(), CreatedBy: "paul@keymail.dev", |
| + }); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| // Astra plan 6 R1 #6: a customer-fleet box's report counts. Its held host |
| // shows held on the list and on the instance page, and the retry door takes |
| -// the retry, through the one loader all three share. |
| +// the retry, through the one loader all three share. The record is placed on |
| +// the fleet, so an unreadable snapshot there makes the list incomplete. |
| func TestFleetBoxHoldShowsOnTheListPageAndRetryDoor(t *testing.T) { |
| fx := newConsoleFixture(t) |
| fx.addRoute(t, registry.Route{Host: heldHost, App: "kass", Account: fx.sqid, Kind: "instance", Status: "ready", Version: "v2", Channel: "stable"}) |
| - sibling(t, fx, "kass", heldHost, "stable") |
| + placedRecord(t, fx, "kass", heldHost, registry.PlacementTarget("pi")) |
| label := writeFleetHeldBox(t, fx, "pi", fx.sqid, heldHost, "kass", "v2") |
| token := cliToken(t, fx, "paul@keymail.dev", "code-fleet-held") |
| @@ -89,7 +104,8 @@ func TestFleetBoxHoldShowsOnTheListPageAndRetryDoor(t *testing.T) { |
| t.Fatalf("retry = %d %s, want 200 for v2", code, body) |
| } |
| - // A fleet box snapshot that cannot be read makes the join incomplete. |
| + // A snapshot that cannot be read on the fleet the record is placed on |
| + // makes the join incomplete. |
| if _, err := fx.rel.Put(context.Background(), "fleets/pi/status/boxes/broken.json", []byte("{not json"), ""); err != nil { |
| t.Fatal(err) |
| } |
| @@ -142,3 +158,50 @@ func TestJoinFleetRolloutsNeverOverrideAPlatformBox(t *testing.T) { |
| t.Fatalf("h on an empty join = %+v, want the fleet box's report", got) |
| } |
| } |
| + |
| +// Astra plan 6 R2 (new Medium): completeness covers only the sources that |
| +// could hold this app's reports. The account has a fleet whose snapshot is |
| +// corrupt, but none of the app's instances is placed there; the app runs on |
| +// healthy platform boxes with no reports (flag off). The list must not say |
| +// rollout_incomplete, so the deploy watch falls back to today's URL watch. |
| +func TestAnUnrelatedUnreadableFleetDoesNotMakeTheListIncomplete(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + placedRecord(t, fx, "kass", heldHost, "ie") |
| + reg, err := registry.Open(fx.regPath) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer reg.Close() |
| + if err := reg.UpsertFleet(registry.Fleet{Name: "spare", Account: fx.sqid, Bucket: "spare-bucket"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := fx.rel.Put(context.Background(), "fleets/spare/status/boxes/broken.json", []byte("{not json"), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + box := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| + box.Metrics = &status.BoxMetrics{RouteVersions: map[string]string{heldHost: "v1"}} |
| + if err := status.WriteBox(context.Background(), fx.rel, box); err != nil { |
| + t.Fatal(err) |
| + } |
| + token := cliToken(t, fx, "paul@keymail.dev", "code-fleet-unrelated") |
| + ro, incomplete := listRollout(t, fx, token, "kass", heldHost) |
| + if ro != nil || incomplete { |
| + t.Fatalf("rollout %+v, incomplete %v: an unrelated fleet's unreadable snapshot blocked the legacy watch", ro, incomplete) |
| + } |
| +} |
| + |
| +// The fleets that could hold an app's reports: where its records are placed, |
| +// and both ends of a move in flight. A platform region names none. |
| +func TestRecordFleetsNamesPlacementsAndMoveEnds(t *testing.T) { |
| + got := recordFleets([]provision.Record{ |
| + {Host: "a", Region: "ie"}, |
| + {Host: "b", Region: registry.PlacementTarget("pi")}, |
| + {Host: "c", Region: "ie", Move: &provision.MoveState{From: "ie", To: registry.PlacementTarget("zero")}}, |
| + }) |
| + if len(got) != 2 || !got["pi"] || !got["zero"] { |
| + t.Fatalf("fleets = %v, want pi and zero", got) |
| + } |
| + if got := recordFleets(nil); got == nil || len(got) != 0 { |
| + t.Fatalf("no records = %v, want an empty, non-nil set", got) |
| + } |
| +} |
diff --git a/internal/provision/plan9_b8_census_test.go b/internal/provision/plan9_b8_census_test.go| index 7d40cceee..7e1f3ae41 100644 |
| --- a/internal/provision/plan9_b8_census_test.go |
| +++ b/internal/provision/plan9_b8_census_test.go |
| @@ -584,7 +584,7 @@ var plan9B8CensusInventory = []censusEntry{ |
| {"internal/console/apiinstancemove.go", "Server.handleAPIInstanceMoveRequest", "record-writer", "key:SetRecordMove,lit:instances", "708ef4614cf5fd9c1ad382cb79da55af0b1192a235fd719765affda92aaf1668"}, |
| {"internal/console/apiinstancemove.go", "Server.registerMovesPhase1Routes", "key-site", "lit:instances", "5af589d288ba22856f631481c40cc001aac342ea74cee6ee0464ec77d0320bbc"}, |
| {"internal/console/apiinstances.go", "Server.instanceManifest", "key-site", "lit:instances", "e6160ffe84abda9163527ccb33e2ffff2109b35908adb2f3a754534325658fdf"}, |
| - {"internal/console/apiinstances.go", "Server.handleAPIInstanceList", "key-site", "lit:instances", "e1ef38cb728c50625690ce16ffd49c4548d6cb2e5e5277197611cf7cbfd6df47"}, |
| + {"internal/console/apiinstances.go", "Server.handleAPIInstanceList", "key-site", "lit:instances", "73a40e68abf796870589df197fc72697bfe201fe0b3ca6c14829f1fd5761f758"}, |
| {"internal/console/apiinstances.go", "Server.handleAPIInstanceCreate", "record-writer", "key:CreateRecord", "20f862446f119f25d7d72eab6edd1b7cfb1528afd5b4c352e0cb8442f5080b13"}, |
| {"internal/console/apiinstances.go", "Server.handleAPIInstanceUpdate", "record-writer", "key:UpdateRecord", "4b792c410399631eb3b38c78829df85dda045651bbc32df769767cf52f015b6d"}, |
| {"internal/console/apiscan.go", "Server.writeScanRecord", "nonrecord", "sel:Put", "c3c092f4b7be9bf6bf09d0fe1e2c9fe52892ee97e57f6c7ab964b85fe36fb5c6"}, |
| @@ -617,7 +617,7 @@ var plan9B8CensusInventory = []censusEntry{ |
| {"internal/console/geoippublish.go", "geoipPublisher.ExpireIfDue", "nonrecord", "sel:Put", "a7d4f24a25c7bda40f18268d34a99203b58aa07bc10eda0db2306b54cb08302b"}, |
| {"internal/console/hostreports.go", "putHostReport", "nonrecord", "sel:Put", "d8ba8c77b5f0ee5bcde5c11f4a608cf2bd2491c36b8cb9bf26753a230d485ad4"}, |
| {"internal/console/infrapass.go", "Server.putInventory", "nonrecord", "sel:Put", "c5d930ec178ee206d25194a2b265738ac42c6f00bd9b73626e4d46d769d370ad"}, |
| - {"internal/console/instancepage.go", "Server.handleAppInstance", "key-site", "lit:instances", "29e4e5717910dd5ea943ad3a0bef7ec2ef835b83e8d0858c9a29c4ea5940721a"}, |
| + {"internal/console/instancepage.go", "Server.handleAppInstance", "key-site", "lit:instances", "33e23b2711aaa1d9c8afbbeabd42d9001fd8c35679d4119e22957eed273e2258"}, |
| {"internal/console/relayops.go", "Server.relayAcquire", "nonrecord", "sel:Put", "3c757e1c92f1c0ce41c3b5f00aa1397ce5a0c4faabc2f176b23355cf5a41f5fb"}, |
| {"internal/console/relayops.go", "Server.relayPublishOnce", "nonrecord", "sel:Put", "783a3c20337914e08f92be783f6b48ab14288437a8043c0d5bf29f281a7a9b1d"}, |
| {"internal/console/relayops.go", "Server.relayRotate", "nonrecord", "sel:Put", "58e0090a4990e2d5e793cec2ec7bcbfb3626b4043307f64484a55766099e1434"}, |