amadan
console: seedRegionReady joins platform heartbeats only, logs the unconfirmed box (Plan 3 Task 5 review)
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2 files changed,
+122
−8
internal/console/seedfloor.go+31 −7internal/console/seedfloor_test.go+91 −1
CI passed — run details
diff --git a/internal/console/seedfloor.go b/internal/console/seedfloor.go| index 7d734d2e9..80ecf200e 100644 |
| --- a/internal/console/seedfloor.go |
| +++ b/internal/console/seedfloor.go |
| @@ -18,7 +18,8 @@ import ( |
| // console runs without claim control, the seeds floor is published, and |
| // every box in region has armed it, excluded or vouched for its |
| // carlos.prev, runs nested, and has no claim control. region is the record |
| -// vocabulary ("" is the home region). nil means go ahead. |
| +// vocabulary ("" is the home region); the home region's name is accepted |
| +// too. nil means go ahead. |
| // |
| // oneBox is the import door's extra rule: the swap, its journal and the |
| // rollback live on the one box holding the legacy row, and a second box in |
| @@ -31,6 +32,7 @@ import ( |
| // box whose own local floor is unreadable publishes "" for its floor, and |
| // must never be read as ready. |
| func (s *Server) seedRegionReady(ctx context.Context, region string, oneBox bool) *doorRefusal { |
| + region = s.homeRegionIn(region) |
| if s.cfg.HostClaim { |
| return &doorRefusal{http.StatusConflict, "This console runs with claim control on. Seeded instances are not supported here yet."} |
| } |
| @@ -87,24 +89,46 @@ func (s *Server) seedRegionReady(ctx context.Context, region string, oneBox bool |
| if len(below) > 0 { |
| return &doorRefusal{http.StatusConflict, fmt.Sprintf("%s in %s are below the seeds floor. Roll the platform there, then publish the seeds floor again.", boxList(below), name)} |
| } |
| - boxes, _, err := s.systemBoxes(ctx, false) |
| + // The platform's own heartbeats, the set complete() vouched for. Never |
| + // systemBoxes: it appends every customer fleet's heartbeats, and a fleet |
| + // box sharing a label could stand in for a platform box. The join is |
| + // the edges file's box against the heartbeat's Label, the pair |
| + // readLiveEdgesSnapshot joins on. A label two heartbeats claim is |
| + // ambiguous, so unconfirmed. |
| + boxes, err := status.LoadBoxes(ctx, s.cfg.Releases) |
| if err != nil { |
| log.Printf("console: seeds readiness: boxes: %v", err) |
| return unconfirmed |
| } |
| - // The join is the edges file's box against the heartbeat's Label, the |
| - // pair readLiveEdgesSnapshot joins on. A region box with no heartbeat, |
| - // a stale one, or one with no layout report is unconfirmed, never |
| - // nested. |
| byLabel := map[string]status.Box{} |
| + dup := map[string]bool{} |
| for _, b := range boxes { |
| + if _, seen := byLabel[b.Label]; seen { |
| + dup[b.Label] = true |
| + } |
| byLabel[b.Label] = b |
| } |
| + // A region box with no heartbeat, a stale one, or one with no layout |
| + // report is unconfirmed, never nested. The refusal stays generic; the |
| + // log names the box, since a retired box whose edges file nobody |
| + // deleted refuses here until someone does. |
| now := time.Now() |
| var flat, claimed []string |
| for _, label := range inRegion { |
| b, ok := byLabel[label] |
| - if !ok || status.BoxStale(b, now, status.SnapshotPeriod) || b.Metrics == nil || b.Metrics.Layout == nil { |
| + why := "" |
| + switch { |
| + case !ok: |
| + why = "no heartbeat" |
| + case dup[label]: |
| + why = "more than one heartbeat with this label" |
| + case status.BoxStale(b, now, status.SnapshotPeriod): |
| + why = "stale heartbeat, generated " + b.GeneratedAt.UTC().Format(time.RFC3339) |
| + case b.Metrics == nil || b.Metrics.Layout == nil: |
| + why = "no layout report" |
| + } |
| + if why != "" { |
| + log.Printf("console: seeds readiness: box %q in region %q is unconfirmed: %s", label, name, why) |
| return unconfirmed |
| } |
| if b.Metrics.Layout.Mode != "nested" { |
diff --git a/internal/console/seedfloor_test.go b/internal/console/seedfloor_test.go| index 8379d18ed..81a472228 100644 |
| --- a/internal/console/seedfloor_test.go |
| +++ b/internal/console/seedfloor_test.go |
| @@ -2,6 +2,7 @@ package console |
| import ( |
| "context" |
| + "encoding/json" |
| "errors" |
| "net/http" |
| "os" |
| @@ -10,6 +11,8 @@ import ( |
| "testing" |
| "time" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| "github.com/carlosframework/platform/internal/seeds" |
| "github.com/carlosframework/platform/internal/status" |
| ) |
| @@ -39,6 +42,11 @@ func putSeedEdges(t *testing.T, fx *consoleFixture, label, body string) { |
| // putSeedHeartbeat writes the box's status heartbeat as of at; mode "" is |
| // a box with no layout report. |
| func putSeedHeartbeat(t *testing.T, fx *consoleFixture, label, mode string, claim bool, at time.Time) { |
| + t.Helper() |
| + putSeedHeartbeatIn(t, fx.rel, label, mode, claim, at) |
| +} |
| + |
| +func putSeedHeartbeatIn(t *testing.T, store release.Store, label, mode string, claim bool, at time.Time) { |
| t.Helper() |
| var layout *status.LayoutReport |
| if mode != "" { |
| @@ -48,7 +56,7 @@ func putSeedHeartbeat(t *testing.T, fx *consoleFixture, label, mode string, clai |
| // layout report would publish no metrics at all. |
| b := status.CollectBox(nil, status.Probes{Release: "v0.27.1", Layout: layout, HostClaimEnabled: claim}, at) |
| b.Label = label |
| - if err := status.WriteBox(context.Background(), fx.rel, b); err != nil { |
| + if err := status.WriteBox(context.Background(), store, b); err != nil { |
| t.Fatal(err) |
| } |
| } |
| @@ -230,3 +238,85 @@ func TestSeedRegionReadyNamesBoxesAndRegion(t *testing.T) { |
| t.Fatalf("an import into three boxes: %+v", ref) |
| } |
| } |
| + |
| +// A customer fleet's heartbeat is not a platform box's: a fresh, nested, |
| +// claim-free fleet heartbeat under the same label cannot stand in for a |
| +// stale platform box. |
| +func TestSeedRegionReadyIgnoresFleetHeartbeats(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + publishSeedsFloorFx(t, fx, "v0.27.0") |
| + seedReadyBox(t, fx, "kass-1", "", 1, "v0.27.0", "v0.27.0", "nested", false) |
| + putSeedHeartbeat(t, fx, "kass-1", "nested", false, time.Now().Add(-time.Hour)) |
| + reg, err := registry.Open(fx.regPath) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { reg.Close() }) |
| + if err := reg.UpsertFleet(registry.Fleet{Name: "pi", Account: fx.sqid, Bucket: "pi-bucket"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + putSeedHeartbeatIn(t, release.NewPrefixStore(fx.rel, "fleets/pi/"), "kass-1", "nested", false, time.Now()) |
| + if ref := fx.srv.seedRegionReady(ctx, "", false); ref == nil || ref.Status != http.StatusServiceUnavailable { |
| + t.Fatalf("a fleet heartbeat covered a stale platform box: %+v", ref) |
| + } |
| +} |
| + |
| +// Two platform heartbeats claiming one label are ambiguous: unconfirmed, |
| +// whichever the read happens to see last. |
| +func TestSeedRegionReadyRefusesADuplicateLabel(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + publishSeedsFloorFx(t, fx, "v0.27.0") |
| + seedReadyBox(t, fx, "kass-1", "", 1, "v0.27.0", "v0.27.0", "nested", false) |
| + b := status.CollectBox(nil, status.Probes{Release: "v0.27.1", Layout: &status.LayoutReport{Mode: "flat"}}, time.Now()) |
| + b.Label = "kass-1" |
| + body, err := json.Marshal(b) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := fx.rel.Put(ctx, "status/boxes/kass-1-old.json", body, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + if ref := fx.srv.seedRegionReady(ctx, "", false); ref == nil || ref.Status != http.StatusServiceUnavailable { |
| + t.Fatalf("two heartbeats for kass-1: %+v", ref) |
| + } |
| +} |
| + |
| +// The floor's own min protocol binds too, when it is above the one this |
| +// console requires. |
| +func TestSeedRegionReadyHonoursTheFloorsProtocol(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + if err := seeds.PublishFloor(ctx, fx.rel, seeds.Floor{Min: seeds.RequiredProtocol + 1, MinRelease: "v0.27.0"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + seedReadyBox(t, fx, "kass-1", "", seeds.RequiredProtocol, "v0.27.0", "v0.27.0", "nested", false) |
| + want := "kass-1 in the home region are below the seeds floor. Roll the platform there, then publish the seeds floor again." |
| + if ref := fx.srv.seedRegionReady(ctx, "", false); ref == nil || ref.Status != http.StatusConflict || ref.Msg != want { |
| + t.Fatalf("a box below the floor's protocol: %+v", ref) |
| + } |
| + seedReadyBox(t, fx, "kass-1", "", seeds.RequiredProtocol+1, "v0.27.0", "v0.27.0", "nested", false) |
| + if ref := fx.srv.seedRegionReady(ctx, "", false); ref != nil { |
| + t.Fatalf("a box at the floor's protocol: %+v", ref) |
| + } |
| +} |
| + |
| +// A caller may name the home region rather than pass the record's "". |
| +func TestSeedRegionReadyAcceptsTheHomeRegionsName(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + fx.srv.cfg.HomeRegion = "eu" |
| + ctx := context.Background() |
| + publishSeedsFloorFx(t, fx, "v0.27.0") |
| + seedReadyBox(t, fx, "kass-1", "", 1, "v0.27.0", "v0.27.0", "nested", false) |
| + for _, region := range []string{"", "eu"} { |
| + if ref := fx.srv.seedRegionReady(ctx, region, false); ref != nil { |
| + t.Errorf("region %q: %+v", region, ref) |
| + } |
| + } |
| + seedReadyBox(t, fx, "kass-1", "", 1, "v0.27.0", "v0.27.0", "flat", false) |
| + want := "kass-1 in eu do not run instances nested yet. A seeded instance needs every box there nested." |
| + if ref := fx.srv.seedRegionReady(ctx, "eu", false); ref == nil || ref.Msg != want { |
| + t.Fatalf("region by name, a flat box: %+v", ref) |
| + } |
| +} |