amadan

carlos / platform Public

console: seedRegionReady, the floor, layout and claim check before any seed state (Plan 3 Task 5)

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude e6a65ae7ec667da9976871335b9268356c107159 parent a753594
3 files changed, +364 −2
  • approved_seeded_copy_test.go +2 −2
  • internal/console/seedfloor.go +130 −0
  • internal/console/seedfloor_test.go +232 −0
diff --git a/approved_seeded_copy_test.go b/approved_seeded_copy_test.go
index a4bc3a41b..521a40e10 100644
--- a/approved_seeded_copy_test.go
+++ b/approved_seeded_copy_test.go
@@ -32,8 +32,8 @@ var seededCopyIDs = func() []string {
}()
// seededWiredIDs grows as the plans wire rows. Plan 1 wires none; Plan 3
-// Task 3 wires SD-N1.
-var seededWiredIDs = []string{"SD-N1"}
+// Task 3 wires SD-N1, Task 5 SD-N2 to SD-N8 and SD-N113.
+var seededWiredIDs = []string{"SD-N1", "SD-N2", "SD-N3", "SD-N4", "SD-N5", "SD-N6", "SD-N7", "SD-N8", "SD-N113"}
func mustParseSeededRecord(t *testing.T) []copyRow {
t.Helper()
diff --git a/internal/console/seedfloor.go b/internal/console/seedfloor.go
new file mode 100644
index 000000000..7d734d2e9
--- /dev/null
+++ b/internal/console/seedfloor.go
@@ -0,0 +1,130 @@
+package console
+
+import (
+ "context"
+ "fmt"
+ "log"
+ "net/http"
+ "sort"
+ "strings"
+ "time"
+
+ "github.com/carlosframework/platform/internal/seeds"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+// seedRegionReady is the check both seed doors make before writing any seed
+// state, even awaiting-upload (seeded-instances spec §3.3, §6.3): this
+// console runs without claim control, the seeds floor is published, and
+// every box in region has armed it, excluded or vouched for its
+// carlos.prev, runs nested, and has no claim control. region is the record
+// vocabulary ("" is the home region). nil means go ahead.
+//
+// oneBox is the import door's extra rule: the swap, its journal and the
+// rollback live on the one box holding the legacy row, and a second box in
+// the region could finish an aborted import without its journal while the
+// swapping box is still rolling back. Until a multi-box design exists, an
+// import refuses a region with more than one box.
+//
+// Every missing, zero or unparseable field is NOT ready. An old box's edges
+// file reads as zeros, so its protocol is checked before anything else: a
+// box whose own local floor is unreadable publishes "" for its floor, and
+// must never be read as ready.
+func (s *Server) seedRegionReady(ctx context.Context, region string, oneBox bool) *doorRefusal {
+ if s.cfg.HostClaim {
+ return &doorRefusal{http.StatusConflict, "This console runs with claim control on. Seeded instances are not supported here yet."}
+ }
+ floor, err := seeds.ReadFloor(ctx, s.cfg.Releases)
+ if err == nil && floor != nil {
+ // A floor PublishFloor would never write (a hand-edited or
+ // truncated object) is as good as unreadable: an empty min_release
+ // would match every box that could not read its own floor.
+ if ok, perr := seeds.ReleaseAtOrAbove(floor.MinRelease, floor.MinRelease); !ok || perr != nil {
+ err = fmt.Errorf("published floor has no usable min_release %q", floor.MinRelease)
+ }
+ }
+ if err != nil {
+ log.Printf("console: seeds floor: %v", err)
+ return &doorRefusal{http.StatusServiceUnavailable, "could not read the seeds floor. Try again."}
+ }
+ if floor == nil {
+ return &doorRefusal{http.StatusConflict, "This deployment has no seeds floor yet. An operator publishes it with carlos system seeds-floor publish."}
+ }
+ minProtocol := max(seeds.RequiredProtocol, floor.Min)
+ name := s.homeRegionOut(region)
+ if name == "" {
+ name = "the home region"
+ }
+ unconfirmed := &doorRefusal{http.StatusServiceUnavailable, fmt.Sprintf("could not confirm every box in %s. Try again in a minute.", name)}
+ snap, err := s.liveEdges(ctx)
+ if err != nil || !snap.complete() {
+ if err != nil {
+ log.Printf("console: seeds readiness: edges: %v", err)
+ }
+ return unconfirmed
+ }
+ var inRegion, below []string
+ for _, f := range snap.Files {
+ if s.homeRegionIn(f.Placement) != region {
+ continue
+ }
+ inRegion = append(inRegion, f.Box)
+ if f.SeedsProtocol < minProtocol {
+ below = append(below, f.Box)
+ continue
+ }
+ prevOK := f.PrevRelease == "none" || f.PrevSeedsAllowedFor == floor.MinRelease
+ if f.SeedsFloorMinRelease != floor.MinRelease || !prevOK {
+ below = append(below, f.Box)
+ }
+ }
+ if len(inRegion) == 0 {
+ return unconfirmed
+ }
+ if oneBox && len(inRegion) > 1 {
+ return &doorRefusal{http.StatusConflict, fmt.Sprintf("%s has more than one box. An import needs a region with one box for now.", name)}
+ }
+ if len(below) > 0 {
+ return &doorRefusal{http.StatusConflict, fmt.Sprintf("%s in %s are below the seeds floor. Roll the platform there, then publish the seeds floor again.", boxList(below), name)}
+ }
+ boxes, _, err := s.systemBoxes(ctx, false)
+ if err != nil {
+ log.Printf("console: seeds readiness: boxes: %v", err)
+ return unconfirmed
+ }
+ // The join is the edges file's box against the heartbeat's Label, the
+ // pair readLiveEdgesSnapshot joins on. A region box with no heartbeat,
+ // a stale one, or one with no layout report is unconfirmed, never
+ // nested.
+ byLabel := map[string]status.Box{}
+ for _, b := range boxes {
+ byLabel[b.Label] = b
+ }
+ now := time.Now()
+ var flat, claimed []string
+ for _, label := range inRegion {
+ b, ok := byLabel[label]
+ if !ok || status.BoxStale(b, now, status.SnapshotPeriod) || b.Metrics == nil || b.Metrics.Layout == nil {
+ return unconfirmed
+ }
+ if b.Metrics.Layout.Mode != "nested" {
+ flat = append(flat, label)
+ }
+ if b.Metrics.HostClaimEnabled {
+ claimed = append(claimed, label)
+ }
+ }
+ if len(flat) > 0 {
+ return &doorRefusal{http.StatusConflict, fmt.Sprintf("%s in %s do not run instances nested yet. A seeded instance needs every box there nested.", boxList(flat), name)}
+ }
+ if len(claimed) > 0 {
+ return &doorRefusal{http.StatusConflict, fmt.Sprintf("%s in %s run with claim control on. Seeded instances are not supported there yet.", boxList(claimed), name)}
+ }
+ return nil
+}
+
+// boxList is the refusals' box names: sorted, comma-joined.
+func boxList(boxes []string) string {
+ sort.Strings(boxes)
+ return strings.Join(boxes, ", ")
+}
diff --git a/internal/console/seedfloor_test.go b/internal/console/seedfloor_test.go
new file mode 100644
index 000000000..8379d18ed
--- /dev/null
+++ b/internal/console/seedfloor_test.go
@@ -0,0 +1,232 @@
+package console
+
+import (
+ "context"
+ "errors"
+ "net/http"
+ "os"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/seeds"
+ "github.com/carlosframework/platform/internal/status"
+)
+
+func seedReadyBox(t *testing.T, fx *consoleFixture, label, placement string, protocol int, floor, prevAllowed, mode string, claim bool) {
+ t.Helper()
+ putSeedEdges(t, fx, label, `{"v":1,"box":"`+label+`","eip":"1.2.3.4","placement":"`+placement+
+ `","hibernation_configured":true,"hosts":{},"seeds_protocol":`+strconv.Itoa(protocol)+
+ `,"seeds_floor_min_release":"`+floor+`","prev_release":"v0.27.1","prev_seeds_allowed_for":"`+prevAllowed+`"}`)
+ putSeedHeartbeat(t, fx, label, mode, claim, time.Now())
+}
+
+// putSeedEdges writes a box's edges file, rewriting it in place: a test
+// moves one box through several states, and putEdges only creates.
+func putSeedEdges(t *testing.T, fx *consoleFixture, label, body string) {
+ t.Helper()
+ key := "control/edges/" + label + ".json"
+ _, etag, err := fx.rel.Get(context.Background(), key)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ t.Fatal(err)
+ }
+ if _, err := fx.rel.Put(context.Background(), key, []byte(body), etag); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// putSeedHeartbeat writes the box's status heartbeat as of at; mode "" is
+// a box with no layout report.
+func putSeedHeartbeat(t *testing.T, fx *consoleFixture, label, mode string, claim bool, at time.Time) {
+ t.Helper()
+ var layout *status.LayoutReport
+ if mode != "" {
+ layout = &status.LayoutReport{Mode: mode}
+ }
+ // A release, as every real heartbeat carries: without one a box with no
+ // layout report would publish no metrics at all.
+ b := status.CollectBox(nil, status.Probes{Release: "v0.27.1", Layout: layout, HostClaimEnabled: claim}, at)
+ b.Label = label
+ if err := status.WriteBox(context.Background(), fx.rel, b); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func publishSeedsFloorFx(t *testing.T, fx *consoleFixture, rel string) {
+ t.Helper()
+ if err := seeds.PublishFloor(context.Background(), fx.rel, seeds.Floor{Min: seeds.RequiredProtocol, MinRelease: rel}); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func TestSeedRegionReady(t *testing.T) {
+ fx := newConsoleFixture(t)
+ ctx := context.Background()
+ if ref := fx.srv.seedRegionReady(ctx, "", false); ref == nil || ref.Status != http.StatusConflict || !strings.Contains(ref.Msg, "no seeds floor yet") {
+ t.Fatalf("no floor: %+v", ref)
+ }
+ publishSeedsFloorFx(t, fx, "v0.27.0")
+ seedReadyBox(t, fx, "kass-1", "", 1, "v0.27.0", "v0.27.0", "nested", false)
+ if ref := fx.srv.seedRegionReady(ctx, "", false); ref != nil {
+ t.Fatalf("a ready region: %+v", ref)
+ }
+ seedReadyBox(t, fx, "kass-2", "", 0, "v0.27.0", "v0.27.0", "nested", false)
+ if ref := fx.srv.seedRegionReady(ctx, "", false); ref == nil || ref.Status != http.StatusConflict || !strings.Contains(ref.Msg, "kass-2") || !strings.Contains(ref.Msg, "below the seeds floor") {
+ t.Fatalf("a box on protocol 0: %+v", ref)
+ }
+ seedReadyBox(t, fx, "kass-2", "", 1, "v0.27.0", "", "nested", false)
+ if ref := fx.srv.seedRegionReady(ctx, "", false); ref == nil || !strings.Contains(ref.Msg, "below the seeds floor") {
+ t.Fatalf("a carlos.prev without a seeds verdict: %+v", ref)
+ }
+ seedReadyBox(t, fx, "kass-2", "", 1, "v0.27.0", "v0.27.0", "flat", false)
+ if ref := fx.srv.seedRegionReady(ctx, "", false); ref == nil || !strings.Contains(ref.Msg, "nested") {
+ t.Fatalf("a flat box: %+v", ref)
+ }
+ seedReadyBox(t, fx, "kass-2", "", 1, "v0.27.0", "v0.27.0", "nested", true)
+ if ref := fx.srv.seedRegionReady(ctx, "", false); ref == nil || !strings.Contains(ref.Msg, "claim control") {
+ t.Fatalf("a claim-armed box: %+v", ref)
+ }
+ fx.srv.cfg.HostClaim = true
+ if ref := fx.srv.seedRegionReady(ctx, "", false); ref == nil || !strings.Contains(ref.Msg, "This console runs with claim control on") {
+ t.Fatalf("a claim console: %+v", ref)
+ }
+}
+
+func TestSeedRegionReadyOneBoxForImports(t *testing.T) {
+ fx := newConsoleFixture(t)
+ publishSeedsFloorFx(t, fx, "v0.27.0")
+ seedReadyBox(t, fx, "kass-1", "", 1, "v0.27.0", "v0.27.0", "nested", false)
+ seedReadyBox(t, fx, "kass-2", "", 1, "v0.27.0", "v0.27.0", "nested", false)
+ if ref := fx.srv.seedRegionReady(context.Background(), "", false); ref != nil {
+ t.Fatalf("two ready boxes, --from: %+v", ref)
+ }
+ if ref := fx.srv.seedRegionReady(context.Background(), "", true); ref == nil || ref.Status != http.StatusConflict || !strings.Contains(ref.Msg, "more than one box") {
+ t.Fatalf("two ready boxes, import: %+v", ref)
+ }
+}
+
+func TestSeedRegionReadyRefusesAnUnconfirmedBox(t *testing.T) {
+ fx := newConsoleFixture(t)
+ publishSeedsFloorFx(t, fx, "v0.27.0")
+ seedReadyBox(t, fx, "kass-1", "", 1, "v0.27.0", "v0.27.0", "", false) // no layout report
+ if ref := fx.srv.seedRegionReady(context.Background(), "", false); ref == nil || ref.Status != http.StatusServiceUnavailable {
+ t.Fatalf("a box with no layout report: %+v", ref)
+ }
+}
+
+func TestSeedRegionReadyIgnoresOtherRegions(t *testing.T) {
+ fx := newConsoleFixture(t)
+ publishSeedsFloorFx(t, fx, "v0.27.0")
+ seedReadyBox(t, fx, "kass-1", "", 1, "v0.27.0", "v0.27.0", "nested", false)
+ seedReadyBox(t, fx, "uk-1", "uk", 0, "", "", "flat", true)
+ if ref := fx.srv.seedRegionReady(context.Background(), "", false); ref != nil {
+ t.Fatalf("a bad box in uk blocked the home region: %+v", ref)
+ }
+ if ref := fx.srv.seedRegionReady(context.Background(), "uk", false); ref == nil {
+ t.Fatal("uk's own bad box did not block uk")
+ }
+}
+
+// Every missing, zero or unparseable field is not ready. An old box's edges
+// file reads as zeros, and a box that could not read its own local floor
+// publishes the protocol but an empty floor: neither may pass, whatever the
+// rest of the file says.
+func TestSeedRegionReadyFailsClosedOnTheEdgesFile(t *testing.T) {
+ fx := newConsoleFixture(t)
+ ctx := context.Background()
+ publishSeedsFloorFx(t, fx, "v0.27.0")
+ seedReadyBox(t, fx, "kass-1", "", 1, "v0.27.0", "v0.27.0", "nested", false)
+ for _, tc := range []struct{ name, body string }{
+ {"an old box", `{"v":1,"box":"kass-2","eip":"1.2.3.4","hosts":{}}`},
+ {"an old box keeping no carlos.prev", `{"v":1,"box":"kass-2","eip":"1.2.3.4","hosts":{},"prev_release":"none","seeds_floor_min_release":"v0.27.0"}`},
+ {"an unreadable local floor", `{"v":1,"box":"kass-2","eip":"1.2.3.4","hosts":{},"seeds_protocol":1,"prev_release":"none"}`},
+ {"a stale local floor", `{"v":1,"box":"kass-2","eip":"1.2.3.4","hosts":{},"seeds_protocol":1,"seeds_floor_min_release":"v0.26.0","prev_release":"none"}`},
+ {"a carlos.prev vouched for an older floor", `{"v":1,"box":"kass-2","eip":"1.2.3.4","hosts":{},"seeds_protocol":1,"seeds_floor_min_release":"v0.27.0","prev_release":"v0.27.1","prev_seeds_allowed_for":"v0.26.0"}`},
+ {"no carlos.prev answer at all", `{"v":1,"box":"kass-2","eip":"1.2.3.4","hosts":{},"seeds_protocol":1,"seeds_floor_min_release":"v0.27.0"}`},
+ } {
+ putSeedEdges(t, fx, "kass-2", tc.body)
+ putSeedHeartbeat(t, fx, "kass-2", "nested", false, time.Now())
+ ref := fx.srv.seedRegionReady(ctx, "", false)
+ want := "kass-2 in the home region are below the seeds floor. Roll the platform there, then publish the seeds floor again."
+ if ref == nil || ref.Status != http.StatusConflict || ref.Msg != want {
+ t.Errorf("%s: %+v", tc.name, ref)
+ }
+ }
+ putSeedEdges(t, fx, "kass-2", `{"v":1,"box":"kass-2","eip":"1.2.3.4","hosts":{},"seeds_protocol":1,"seeds_floor_min_release":"v0.27.0","prev_release":"none"}`)
+ if ref := fx.srv.seedRegionReady(ctx, "", false); ref != nil {
+ t.Fatalf("a box keeping no carlos.prev: %+v", ref)
+ }
+}
+
+// A box in the edges files with no heartbeat at all, or a stale one, is
+// unconfirmed: never nested, never ready.
+func TestSeedRegionReadyRefusesAMissingOrStaleHeartbeat(t *testing.T) {
+ fx := newConsoleFixture(t)
+ ctx := context.Background()
+ publishSeedsFloorFx(t, fx, "v0.27.0")
+ seedReadyBox(t, fx, "kass-1", "", 1, "v0.27.0", "v0.27.0", "nested", false)
+ putSeedEdges(t, fx, "kass-2", `{"v":1,"box":"kass-2","eip":"1.2.3.4","hosts":{},"seeds_protocol":1,"seeds_floor_min_release":"v0.27.0","prev_release":"none"}`)
+ want := "could not confirm every box in the home region. Try again in a minute."
+ if ref := fx.srv.seedRegionReady(ctx, "", false); ref == nil || ref.Status != http.StatusServiceUnavailable || ref.Msg != want {
+ t.Fatalf("a box with no heartbeat: %+v", ref)
+ }
+ putSeedHeartbeat(t, fx, "kass-2", "nested", false, time.Now().Add(-time.Hour))
+ if ref := fx.srv.seedRegionReady(ctx, "", false); ref == nil || ref.Status != http.StatusServiceUnavailable || ref.Msg != want {
+ t.Fatalf("a box with a stale heartbeat: %+v", ref)
+ }
+ putSeedHeartbeat(t, fx, "kass-2", "nested", false, time.Now())
+ if ref := fx.srv.seedRegionReady(ctx, "", false); ref != nil {
+ t.Fatalf("both boxes fresh: %+v", ref)
+ }
+ // A region with no edges file at all is unconfirmed too, named.
+ if ref := fx.srv.seedRegionReady(ctx, "uk", false); ref == nil || ref.Status != http.StatusServiceUnavailable || ref.Msg != "could not confirm every box in uk. Try again in a minute." {
+ t.Fatalf("a region with no box: %+v", ref)
+ }
+}
+
+// A published floor that PublishFloor would never write is unreadable, not
+// a floor: an empty min_release would match every box that could not read
+// its own.
+func TestSeedRegionReadyRefusesAnUnusableFloor(t *testing.T) {
+ fx := newConsoleFixture(t)
+ ctx := context.Background()
+ seedReadyBox(t, fx, "kass-1", "", 1, "", "", "nested", false)
+ for _, body := range []string{`{"min":1,"min_release":""}`, `{"min":1}`, `{"min":1,"min_release":"next"}`, `not json`} {
+ _, etag, err := fx.rel.Get(ctx, seeds.FloorKey)
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ t.Fatal(err)
+ }
+ if _, err := fx.rel.Put(ctx, seeds.FloorKey, []byte(body), etag); err != nil {
+ t.Fatal(err)
+ }
+ if ref := fx.srv.seedRegionReady(ctx, "", false); ref == nil || ref.Status != http.StatusServiceUnavailable || ref.Msg != "could not read the seeds floor. Try again." {
+ t.Errorf("floor %s: %+v", body, ref)
+ }
+ }
+}
+
+// The refusals name every box they found, sorted, and the region by its
+// name: the configured home region's, not the record's "".
+func TestSeedRegionReadyNamesBoxesAndRegion(t *testing.T) {
+ fx := newConsoleFixture(t)
+ fx.srv.cfg.HomeRegion = "eu"
+ ctx := context.Background()
+ publishSeedsFloorFx(t, fx, "v0.27.0")
+ seedReadyBox(t, fx, "kass-3", "eu", 1, "v0.27.0", "v0.27.0", "flat", false)
+ seedReadyBox(t, fx, "kass-1", "eu", 1, "v0.27.0", "v0.27.0", "flat", false)
+ seedReadyBox(t, fx, "kass-2", "", 1, "v0.27.0", "v0.27.0", "nested", true)
+ want := "kass-1, kass-3 in eu do not run instances nested yet. A seeded instance needs every box there nested."
+ if ref := fx.srv.seedRegionReady(ctx, "", false); ref == nil || ref.Status != http.StatusConflict || ref.Msg != want {
+ t.Fatalf("two flat boxes: %+v", ref)
+ }
+ seedReadyBox(t, fx, "kass-3", "eu", 1, "v0.27.0", "v0.27.0", "nested", true)
+ seedReadyBox(t, fx, "kass-1", "eu", 1, "v0.27.0", "v0.27.0", "nested", false)
+ want = "kass-2, kass-3 in eu run with claim control on. Seeded instances are not supported there yet."
+ if ref := fx.srv.seedRegionReady(ctx, "", false); ref == nil || ref.Status != http.StatusConflict || ref.Msg != want {
+ t.Fatalf("two claim boxes: %+v", ref)
+ }
+ if ref := fx.srv.seedRegionReady(ctx, "", true); ref == nil || ref.Status != http.StatusConflict || ref.Msg != "eu has more than one box. An import needs a region with one box for now." {
+ t.Fatalf("an import into three boxes: %+v", ref)
+ }
+}