boxes publish lifetimes-1 and purge nothing claims-off; the cutover door refuses below the box floor, and the watch pages a regression hourly
A claims-off box purges nothing from this release on: domains.Reconcile's 1d auto-purge runs only for HostClaim, which still requests the claim's purge (TestBoxesNoLongerPurge); a box's check-then-delete can always pause between the two, so no fence on a box can protect a successor of the app's name. Every box of this release publishes lifetimes-1 in its edges file (edgesFileCapabilities, always on; region-ha-1 still follows its flag). The cutover door, before the marker is first written, reads every edges file of the deployment and refuses 409 "box <label> does not publish lifetimes-1; every box must run this release before the cutover" while any lacks it, and 409 "box <label> has not reported in 10 minutes" while any such box's status report is missing or older than ten minutes: liveness from the status report, never from the edges file, which a healthy box does not rewrite. An unreadable edges file is the offline move's "could not all be read" 409. After the cutover the watch reads the files every five minutes and pages "box <label> regressed below the lifetimes floor" once an hour per box.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
10 files changed,
+387
−71
internal/console/lifetimes_cutover.go+14 −0internal/console/lifetimes_floor.go+137 −0internal/console/lifetimes_floor_test.go+114 −0internal/console/seedfloor_test.go+1 −1internal/console/server.go+3 −1internal/domains/reconcile.go+13 −19internal/domains/reconcile_test.go+48 −34internal/host/edgesfile.go+12 −5internal/host/edgesfile_test.go+35 −3internal/host/movesoffline_test.go+10 −8
diff --git a/internal/console/lifetimes_cutover.go b/internal/console/lifetimes_cutover.go| index d9734cf35..8feb44de0 100644 |
| --- a/internal/console/lifetimes_cutover.go |
| +++ b/internal/console/lifetimes_cutover.go |
| @@ -467,6 +467,9 @@ func (s *Server) lifetimesWatchOnce(ctx context.Context) { |
| log.Printf("console: lifetimes: %s -> %s", was, is) |
| } |
| } |
| + if g.current() == cutoverOn { |
| + s.lifetimesFloorWatch(ctx, now) |
| + } |
| s.materializeAppsOnce(ctx) |
| } |
| @@ -523,6 +526,17 @@ func (s *Server) handleAPILifetimesCutover(w http.ResponseWriter, r *http.Reques |
| } |
| g := s.cutover |
| now := g.clock() |
| + // The box floor (lifetimes_floor.go), before the marker is first |
| + // written; once it is, the door answers the status. |
| + if st := g.current(); st == cutoverBefore || st == cutoverUnknown { |
| + fctx, fcancel := context.WithTimeout(r.Context(), instanceStoreTimeout) |
| + ref := s.lifetimesBoxFloor(fctx, now) |
| + fcancel() |
| + if ref != nil { |
| + jsonError(w, ref.Status, ref.Msg) |
| + return |
| + } |
| + } |
| m := cutoverMarker{At: now.UTC(), By: addr} |
| body, err := json.Marshal(m) |
| if err != nil { |
diff --git a/internal/console/lifetimes_floor.go b/internal/console/lifetimes_floor.go| new file mode 100644 |
| index 000000000..b1bed2ade |
| --- /dev/null |
| +++ b/internal/console/lifetimes_floor.go |
| @@ -0,0 +1,137 @@ |
| +package console |
| + |
| +import ( |
| + "context" |
| + "fmt" |
| + "log" |
| + "net/http" |
| + "slices" |
| + "sort" |
| + "sync" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/alerting" |
| + "github.com/carlosframework/platform/internal/status" |
| +) |
| + |
| +// The box floor of the lifetimes cutover (self-serve moves plan 1, Task 4). |
| +// A box running this release publishes lifetimes-1 in its edges file's |
| +// capabilities (internal/host edgesFileCapabilities): it purges nothing and |
| +// leaves every instance lifetime to the console. The cutover door refuses |
| +// while any edges file of the deployment lacks it, or while any such box's |
| +// status report (status/boxes/<label>.json) is missing or older than ten |
| +// minutes: freshness comes from the status report, never from the edges |
| +// file, which a healthy box does not rewrite while nothing changes. A box |
| +// with no edges file is not a box of the fleet. After the cutover the watch |
| +// pages once an hour per box whose file lacks the capability. |
| + |
| +const ( |
| + // capLifetimes1 is the edges-file capability of the box floor. |
| + capLifetimes1 = "lifetimes-1" |
| + // lifetimesBoxFresh is how recent a box's status report must be. |
| + lifetimesBoxFresh = 10 * time.Minute |
| + // lifetimesFloorEvery is how often the watch reads the fleet after the |
| + // cutover; lifetimesFloorPageEvery how often it pages per box. |
| + lifetimesFloorEvery = 5 * time.Minute |
| + lifetimesFloorPageEvery = time.Hour |
| + |
| + msgLifetimesFleetUnread = "the fleet's edges files could not all be read. try again" |
| +) |
| + |
| +// lifetimesPage is how the floor watch pages. A TEST SEAM. |
| +var lifetimesPage = alerting.Page |
| + |
| +func msgLifetimesBoxBelowFloor(box string) string { |
| + return fmt.Sprintf("box %s does not publish lifetimes-1; every box must run this release before the cutover", box) |
| +} |
| + |
| +func msgLifetimesBoxSilent(box string) string { |
| + return fmt.Sprintf("box %s has not reported in 10 minutes", box) |
| +} |
| + |
| +func pageLifetimesRegressed(box string) string { |
| + return fmt.Sprintf("box %s regressed below the lifetimes floor", box) |
| +} |
| + |
| +// lifetimesFloorState is the watch's memory, kept on the Server. |
| +type lifetimesFloorState struct { |
| + floorMu sync.Mutex |
| + floorReadAt time.Time |
| + floorPagedAt map[string]time.Time |
| +} |
| + |
| +// lifetimesBoxFloor checks every edges file of the deployment and its box's |
| +// status report at now. nil is accepted; otherwise the refusal to answer. |
| +// The status report is joined by its label to the file's box, the join every |
| +// fleet reader makes (status reports carry no other box identity). |
| +func (s *Server) lifetimesBoxFloor(ctx context.Context, now time.Time) *doorRefusal { |
| + snap, err := readEdgesSnapshot(ctx, s.cfg.Releases) |
| + if err != nil || !snap.Complete { |
| + log.Printf("console: lifetimes: the box floor: the edges files are not complete evidence (error %v)", err) |
| + return &doorRefusal{http.StatusConflict, msgLifetimesFleetUnread} |
| + } |
| + files := snap.Files |
| + sort.Slice(files, func(i, j int) bool { return files[i].Box < files[j].Box }) |
| + for _, f := range files { |
| + if !slices.Contains(f.Capabilities, capLifetimes1) { |
| + return &doorRefusal{http.StatusConflict, msgLifetimesBoxBelowFloor(f.Box)} |
| + } |
| + } |
| + boxes, complete, err := status.LoadBoxesComplete(ctx, s.cfg.Releases) |
| + if err != nil || !complete { |
| + log.Printf("console: lifetimes: the box floor: the status reports are not complete evidence (complete %v, error %v)", complete, err) |
| + } |
| + reported := map[string]time.Time{} |
| + for _, b := range boxes { |
| + reported[b.Label] = b.GeneratedAt |
| + } |
| + for _, f := range files { |
| + at, ok := reported[f.Box] |
| + if !ok || at.IsZero() || now.Sub(at) > lifetimesBoxFresh { |
| + return &doorRefusal{http.StatusConflict, msgLifetimesBoxSilent(f.Box)} |
| + } |
| + } |
| + return nil |
| +} |
| + |
| +// lifetimesFloorWatch is the floor's check after the cutover, from the |
| +// cutover watch: at most every lifetimesFloorEvery it reads the edges files, |
| +// and pages once per lifetimesFloorPageEvery per box whose file lacks the |
| +// capability. |
| +func (s *Server) lifetimesFloorWatch(ctx context.Context, now time.Time) { |
| + s.floorMu.Lock() |
| + if !s.floorReadAt.IsZero() && now.Sub(s.floorReadAt) < lifetimesFloorEvery { |
| + s.floorMu.Unlock() |
| + return |
| + } |
| + s.floorReadAt = now |
| + s.floorMu.Unlock() |
| + snap, err := readEdgesSnapshot(ctx, s.cfg.Releases) |
| + if err != nil { |
| + log.Printf("console: lifetimes: the box floor watch: %v", err) |
| + return |
| + } |
| + for _, f := range snap.Files { |
| + if slices.Contains(f.Capabilities, capLifetimes1) { |
| + continue |
| + } |
| + s.floorMu.Lock() |
| + if s.floorPagedAt == nil { |
| + s.floorPagedAt = map[string]time.Time{} |
| + } |
| + last, paged := s.floorPagedAt[f.Box] |
| + due := !paged || now.Sub(last) >= lifetimesFloorPageEvery |
| + if due { |
| + s.floorPagedAt[f.Box] = now |
| + } |
| + s.floorMu.Unlock() |
| + if !due { |
| + continue |
| + } |
| + subject := pageLifetimesRegressed(f.Box) |
| + log.Printf("console: lifetimes: %s", subject) |
| + if err := lifetimesPage(ctx, alerting.LevelIncident, subject, "its edges file at control/edges/"+f.Box+".json does not list "+capLifetimes1); err != nil { |
| + log.Printf("console: lifetimes: page: %v", err) |
| + } |
| + } |
| +} |
diff --git a/internal/console/lifetimes_floor_test.go b/internal/console/lifetimes_floor_test.go| new file mode 100644 |
| index 000000000..a40acd1db |
| --- /dev/null |
| +++ b/internal/console/lifetimes_floor_test.go |
| @@ -0,0 +1,114 @@ |
| +package console |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "net/http" |
| + "sync" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// The box floor of the lifetimes cutover (lifetimes_floor.go): the door's |
| +// two refusals and the hourly page after the cutover. |
| + |
| +// ---- the box floor ---------------------------------------------------------- |
| + |
| +// writeEdges is an edges file of box as a box running this release publishes |
| +// it with region HA off and unchanged routes: written days ago, never |
| +// rewritten since. |
| +func writeEdges(t *testing.T, st release.Store, box string, caps ...string) { |
| + t.Helper() |
| + body, _ := json.Marshal(map[string]any{"v": 1, "box": box, "box_id": box, "eip": "203.0.113.10", |
| + "generation": "2026-10-01T09:00:00Z", "capabilities": caps, "hosts": map[string]any{}}) |
| + if _, err := st.Put(context.Background(), "control/edges/"+box+".json", body, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +func writeBoxStatus(t *testing.T, st release.Store, label string, at time.Time) { |
| + t.Helper() |
| + body, _ := json.Marshal(map[string]any{"generated_at": at, "label": label, "routes": map[string]any{}, "replication": map[string]any{}}) |
| + if _, err := st.Put(context.Background(), "status/boxes/"+label+".json", body, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +func TestCutoverRefusesBoxesBelowTheFloor(t *testing.T) { |
| + cases := []struct { |
| + name string |
| + arrange func(t *testing.T, l lifetimesFx) |
| + code int |
| + msg string |
| + }{ |
| + {"a box without the capability", func(t *testing.T, l lifetimesFx) { |
| + writeEdges(t, l.fx.rel, "annie", capLifetimes1) |
| + writeEdges(t, l.fx.rel, "bert") |
| + writeBoxStatus(t, l.fx.rel, "annie", l.clock.Now()) |
| + writeBoxStatus(t, l.fx.rel, "bert", l.clock.Now()) |
| + }, http.StatusConflict, "box bert does not publish lifetimes-1; every box must run this release before the cutover"}, |
| + {"a stale status report", func(t *testing.T, l lifetimesFx) { |
| + writeEdges(t, l.fx.rel, "annie", capLifetimes1) |
| + writeBoxStatus(t, l.fx.rel, "annie", l.clock.Now().Add(-11*time.Minute)) |
| + }, http.StatusConflict, "box annie has not reported in 10 minutes"}, |
| + {"a missing status report", func(t *testing.T, l lifetimesFx) { |
| + writeEdges(t, l.fx.rel, "annie", capLifetimes1) |
| + }, http.StatusConflict, "box annie has not reported in 10 minutes"}, |
| + {"an old edges file and a fresh report", func(t *testing.T, l lifetimesFx) { |
| + writeEdges(t, l.fx.rel, "annie", capLifetimes1, "region-ha-1") |
| + writeBoxStatus(t, l.fx.rel, "annie", l.clock.Now().Add(-time.Minute)) |
| + }, http.StatusOK, ""}, |
| + } |
| + for _, c := range cases { |
| + t.Run(c.name, func(t *testing.T) { |
| + l := newLifetimesFx(t) |
| + c.arrange(t, l) |
| + code, _, body := l.cutover(t, l.operator) |
| + if code != c.code || (c.msg != "" && cutoverErrorOf(body) != c.msg) { |
| + t.Fatalf("cutover = %d %s, want %d %q", code, body, c.code, c.msg) |
| + } |
| + _, _, err := l.fx.rel.Get(context.Background(), lifetimesCutoverKey) |
| + if written := err == nil; written != (c.code == http.StatusOK) { |
| + t.Fatalf("marker written = %v, want %v", written, c.code == http.StatusOK) |
| + } |
| + }) |
| + } |
| + t.Run("a file losing the capability after the cutover pages hourly", func(t *testing.T) { |
| + l := newLifetimesFx(t) |
| + writeEdges(t, l.fx.rel, "annie", capLifetimes1) |
| + writeBoxStatus(t, l.fx.rel, "annie", l.clock.Now()) |
| + turnLifetimesOn(t, l.fx.srv, l.clock) |
| + var pages []string |
| + var mu sync.Mutex |
| + old := lifetimesPage |
| + lifetimesPage = func(_ context.Context, _, subject, _ string) error { |
| + mu.Lock() |
| + pages = append(pages, subject) |
| + mu.Unlock() |
| + return nil |
| + } |
| + t.Cleanup(func() { lifetimesPage = old }) |
| + ctx := context.Background() |
| + l.fx.srv.lifetimesWatchOnce(ctx) |
| + writeEdges(t, l.fx.rel, "bert") |
| + l.clock.Advance(lifetimesFloorEvery) |
| + l.fx.srv.lifetimesWatchOnce(ctx) |
| + l.clock.Advance(lifetimesFloorEvery) |
| + l.fx.srv.lifetimesWatchOnce(ctx) // within the hour: no second page |
| + mu.Lock() |
| + got := append([]string(nil), pages...) |
| + mu.Unlock() |
| + if len(got) != 1 || got[0] != "box bert regressed below the lifetimes floor" { |
| + t.Fatalf("pages = %q, want one for bert", got) |
| + } |
| + l.clock.Advance(time.Hour) |
| + l.fx.srv.lifetimesWatchOnce(ctx) |
| + mu.Lock() |
| + defer mu.Unlock() |
| + if len(pages) != 2 { |
| + t.Fatalf("pages = %q, want a second after the hour", pages) |
| + } |
| + }) |
| +} |
diff --git a/internal/console/seedfloor_test.go b/internal/console/seedfloor_test.go| index 11c2ac2df..c11f72c42 100644 |
| --- a/internal/console/seedfloor_test.go |
| +++ b/internal/console/seedfloor_test.go |
| @@ -20,7 +20,7 @@ import ( |
| func seedReadyBox(t *testing.T, fx *consoleFixture, label, placement string, protocol int, floor, prevAllowed, mode string, claim bool) { |
| t.Helper() |
| putSeedEdges(t, fx, label, `{"v":1,"box":"`+label+`","eip":"1.2.3.4","placement":"`+placement+ |
| - `","hibernation_configured":true,"hosts":{},"seeds_protocol":`+strconv.Itoa(protocol)+ |
| + `","hibernation_configured":true,"capabilities":["lifetimes-1"],"hosts":{},"seeds_protocol":`+strconv.Itoa(protocol)+ |
| `,"seeds_floor_min_release":"`+floor+`","prev_release":"v0.27.1","prev_seeds_allowed_for":"`+prevAllowed+`"}`) |
| putSeedHeartbeat(t, fx, label, mode, claim, time.Now()) |
| } |
diff --git a/internal/console/server.go b/internal/console/server.go| index 1c36862d1..fe2d5f9ac 100644 |
| --- a/internal/console/server.go |
| +++ b/internal/console/server.go |
| @@ -1155,8 +1155,10 @@ type Server struct { |
| // written. Zero values are ready. |
| hostLocks sync.Map |
| lifetimeLogged sync.Map |
| - // The trash sweep's clock (lifetimes_purge.go). |
| + // The trash sweep's clock (lifetimes_purge.go) and the box floor |
| + // watch's memory (lifetimes_floor.go). |
| trashSweepState |
| + lifetimesFloorState |
| // relayNow is the relay paths' clock seam: nil means time.Now. Tests |
| // drive lease expiry with it; nothing in production sets it. |
| relayNow func() time.Time |
diff --git a/internal/domains/reconcile.go b/internal/domains/reconcile.go| index cd69ca76a..6f891e83d 100644 |
| --- a/internal/domains/reconcile.go |
| +++ b/internal/domains/reconcile.go |
| @@ -251,29 +251,23 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg |
| } |
| // --- 1d. auto-purge ------------------------------------------------------ |
| - // Trashed longer than TrashCooloff → destroy. The marker's own deleted_at |
| - // is the clock (bucket-side; restore deletes the marker, so there is |
| - // nothing to replay). PurgeApp RE-READS the marker immediately before |
| - // destroying — this listing is only a hint — and re-checks the production |
| - // flag; both guards fail toward keeping data. Idempotent and race-proof: |
| - // concurrent boxes converge because every step tolerates already-gone and |
| - // the marker goes last. |
| - for k, tomb := range trashed { |
| - if cfg.Now().Sub(tomb.DeletedAt) < release.TrashCooloff { |
| - continue |
| - } |
| - if cfg.HostClaim { |
| + // A host-claim box asks for the purge of an app trashed longer than |
| + // TrashCooloff (the marker's own deleted_at is the clock); the claim's |
| + // purge operation does the destroying. A claims-off box purges nothing |
| + // from this release on (self-serve moves plan 1, Task 4): a box's |
| + // check-then-delete can always pause between the check and the delete, |
| + // so nothing on a box can fence its sweep against a successor of the |
| + // app's name, and the console's expiry pass owns the store-driven trash |
| + // sweep instead (console/lifetimes_purge.go). |
| + if cfg.HostClaim { |
| + for k, tomb := range trashed { |
| + if cfg.Now().Sub(tomb.DeletedAt) < release.TrashCooloff { |
| + continue |
| + } |
| if err := release.RequestClaimAutoPurge(ctx, s, k.acct, k.app, cfg.Now()); err != nil { |
| log.Printf("domains: request auto-purge %s/%s: %v", k.acct, k.app, err) |
| } |
| - continue |
| - } |
| - if err := release.PurgeApp(ctx, s, cfg.ReleasePub, k.acct, k.app, cfg.Now()); err != nil { |
| - log.Printf("domains: auto-purge %s/%s: %v", k.acct, k.app, err) |
| - continue |
| } |
| - log.Printf("domains: purged %s/%s — trashed %s, past the %s cooloff", |
| - k.acct, k.app, tomb.DeletedAt.Format(time.RFC3339), release.TrashCooloff) |
| } |
| // --- 2. platform-domain minting ------------------------------------------ |
diff --git a/internal/domains/reconcile_test.go b/internal/domains/reconcile_test.go| index 4e72b2936..27953bd9b 100644 |
| --- a/internal/domains/reconcile_test.go |
| +++ b/internal/domains/reconcile_test.go |
| @@ -1029,41 +1029,55 @@ func appKeysLeft(t *testing.T, s release.Store, acct, app string) []string { |
| return keys |
| } |
| -// Past the cooloff, the pass DESTROYS: the app's objects and, last, its |
| -// marker. The neighbour app in the same account is untouched, and the pass |
| -// converges — a second run over an empty prefix changes nothing and does not |
| -// error. |
| -func TestReconcilePurgesAppsPastTheCooloff(t *testing.T) { |
| +// A claims-off box purges nothing from this release on (self-serve moves |
| +// plan 1, Task 4): past the cooloff the app stays hidden and every object and |
| +// the marker stay, for the console's expiry pass to purge. A host-claim box |
| +// still requests its auto-purge. |
| +func TestBoxesNoLongerPurge(t *testing.T) { |
| _, pub := releaseKeys(t) |
| - s, reg := dirStore(t), regWith(t, |
| - registry.Route{Host: "derp.custom", Kind: "static", App: "derp", |
| - Status: "ready", Channel: "stable", Account: domains.Sqid(793481920157)}, |
| - ) |
| - sqid := writeAccountManifest(t, s, 793481920157, "Kass") |
| - putKey(t, s, "apps/"+sqid+"/derp/channels/stable.json") |
| - putKey(t, s, "apps/"+sqid+"/derp/releases/1.0.0/manifest.json") |
| - putKey(t, s, "apps/"+sqid+"/keep/channels/stable.json") |
| - // Trashed 31 days before the config's clock. |
| - deleted := time.Unix(1754000000, 0).Add(-31 * 24 * time.Hour) |
| - if err := release.TrashApp(ctx, s, sqid, "derp", "paul@keymail.dev", deleted); err != nil { |
| - t.Fatal(err) |
| - } |
| - |
| - c := cfg(t) |
| - c.ReleasePub = pub |
| - for pass := 1; pass <= 2; pass++ { |
| - if err := domains.Reconcile(ctx, s, reg, c); err != nil { |
| - t.Fatalf("pass %d: %v", pass, err) |
| - } |
| - if got := appKeysLeft(t, s, sqid, "derp"); len(got) != 0 { |
| - t.Fatalf("pass %d: purge left objects behind: %v", pass, got) |
| - } |
| - if got := routesFor(t, reg, "derp"); len(got) != 0 { |
| - t.Errorf("pass %d: purged app still routed: %+v", pass, got) |
| - } |
| - if got := appKeysLeft(t, s, sqid, "keep"); len(got) != 1 { |
| - t.Fatalf("pass %d: the purge reached the neighbour app: %v", pass, got) |
| - } |
| + for _, hostClaim := range []bool{false, true} { |
| + t.Run(map[bool]string{false: "claims off", true: "claims on"}[hostClaim], func(t *testing.T) { |
| + s, reg := dirStore(t), regWith(t, |
| + registry.Route{Host: "derp.custom", Kind: "static", App: "derp", |
| + Status: "ready", Channel: "stable", Account: domains.Sqid(793481920157)}, |
| + ) |
| + sqid := writeAccountManifest(t, s, 793481920157, "Kass") |
| + putKey(t, s, "apps/"+sqid+"/derp/channels/stable.json") |
| + putKey(t, s, "apps/"+sqid+"/keep/channels/stable.json") |
| + if hostClaim { |
| + if _, err := s.Put(ctx, "apps/"+sqid+"/derp/admission.json", []byte(`{"v":1,"app_id":"app-derp","state":"open","op":null}`), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + // Trashed 31 days before the config's clock. |
| + deleted := time.Unix(1754000000, 0).Add(-31 * 24 * time.Hour) |
| + if err := release.TrashApp(ctx, s, sqid, "derp", "paul@keymail.dev", deleted); err != nil { |
| + t.Fatal(err) |
| + } |
| + before := appKeysLeft(t, s, sqid, "derp") |
| + c := cfg(t) |
| + c.ReleasePub = pub |
| + c.HostClaim = hostClaim |
| + if err := domains.Reconcile(ctx, s, reg, c); err != nil { |
| + t.Fatal(err) |
| + } |
| + got := appKeysLeft(t, s, sqid, "derp") |
| + if !hostClaim { |
| + if !reflect.DeepEqual(got, before) { |
| + t.Fatalf("a claims-off box changed the trashed app: %v, want %v", got, before) |
| + } |
| + if routes := routesFor(t, reg, "derp"); len(routes) != 0 { |
| + t.Errorf("the trashed app is still routed: %+v", routes) |
| + } |
| + return |
| + } |
| + if !slices.Contains(got, "apps/"+sqid+"/derp/purge-request.json") { |
| + t.Fatalf("a claims-on box made no purge request: %v", got) |
| + } |
| + if !slices.Contains(got, release.AppTombstoneKey(sqid, "derp")) { |
| + t.Fatal("the claims-on box destroyed the marker itself") |
| + } |
| + }) |
| } |
| } |
diff --git a/internal/host/edgesfile.go b/internal/host/edgesfile.go| index e30d70329..271281537 100644 |
| --- a/internal/host/edgesfile.go |
| +++ b/internal/host/edgesfile.go |
| @@ -162,14 +162,21 @@ type edgeHeadroom struct { |
| const capRegionHA1 = "region-ha-1" |
| -// edgesFileCapabilities publishes region-ha-1 only when CARLOS_REGION_HA is |
| -// set to something other than "" or "0". offline-move-1 is added per pass |
| -// by withOfflineCapability. |
| +// capLifetimes1 is the box floor of the console's lifetimes cutover |
| +// (self-serve moves plan 1, Task 4): a box running this release purges |
| +// nothing on a claims-off deployment and leaves every instance lifetime to |
| +// the console. Always published from this release on; the cutover door |
| +// refuses while any edges file lacks it. |
| +const capLifetimes1 = "lifetimes-1" |
| + |
| +// edgesFileCapabilities publishes lifetimes-1 always, and region-ha-1 only |
| +// when CARLOS_REGION_HA is set to something other than "" or "0". |
| +// offline-move-1 is added per pass by withOfflineCapability. |
| func edgesFileCapabilities(getenv func(string) string) []string { |
| if !regionHAEnabled(getenv) { |
| - return nil |
| + return []string{capLifetimes1} |
| } |
| - return []string{capRegionHA1} |
| + return []string{capLifetimes1, capRegionHA1} |
| } |
| // withOfflineCapability is caps with offline-move-1 and offline-stage-1 when |
diff --git a/internal/host/edgesfile_test.go b/internal/host/edgesfile_test.go| index 75f022545..1bd876ff3 100644 |
| --- a/internal/host/edgesfile_test.go |
| +++ b/internal/host/edgesfile_test.go |
| @@ -9,6 +9,7 @@ import ( |
| "log" |
| "os" |
| "path/filepath" |
| + "slices" |
| "strings" |
| "sync" |
| "testing" |
| @@ -1342,17 +1343,48 @@ func TestEdgesFileCapabilityFollowsTheFlag(t *testing.T) { |
| return "" |
| } |
| } |
| - if got := edgesFileCapabilities(env("")); len(got) != 0 { |
| + // lifetimes-1 is the lifetimes cutover's box floor, published always. |
| + if got := edgesFileCapabilities(env("")); !slices.Equal(got, []string{"lifetimes-1"}) { |
| t.Fatalf("unset: %v", got) |
| } |
| - if got := edgesFileCapabilities(env("1")); len(got) != 1 || got[0] != "region-ha-1" { |
| + if got := edgesFileCapabilities(env("1")); !slices.Equal(got, []string{"lifetimes-1", "region-ha-1"}) { |
| t.Fatalf("set: %v", got) |
| } |
| - if got := edgesFileCapabilities(env("0")); len(got) != 0 { |
| + if got := edgesFileCapabilities(env("0")); !slices.Equal(got, []string{"lifetimes-1"}) { |
| t.Fatalf("\"0\" is off: %v", got) |
| } |
| } |
| +// The lifetimes cutover's box floor through the real publisher (self-serve |
| +// moves plan 1, Task 4): with region HA off and nothing changing, the edges |
| +// file carries lifetimes-1 and is written once, so its generation stands |
| +// while the box stays healthy (the console takes liveness from the status |
| +// report instead). |
| +func TestEdgesFilePublishesTheLifetimesFloor(t *testing.T) { |
| + s, reg := release.NewDirStore(t.TempDir()), edgesTestRegistry(t) |
| + seedRoute(t, reg, "app.example.com") |
| + none := func(string) string { return "" } |
| + ident := edgesFile{Box: "annie", BoxID: "annie", Capabilities: edgesFileCapabilities(none)} |
| + var last []byte |
| + if err := edgesFileWrite(context.Background(), s, reg, ident, "203.0.113.10", &last, false); err != nil { |
| + t.Fatal(err) |
| + } |
| + b1, e1, err := s.Get(context.Background(), edgesFileKey("annie")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var f edgesfile.PlacementView |
| + if err := json.Unmarshal(b1, &f); err != nil || !slices.Equal(f.Capabilities, []string{capLifetimes1}) { |
| + t.Fatalf("capabilities = %v (%v), want [lifetimes-1]", f.Capabilities, err) |
| + } |
| + if err := edgesFileWrite(context.Background(), s, reg, ident, "203.0.113.10", &last, false); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, e2, _ := s.Get(context.Background(), edgesFileKey("annie")); e2 != e1 { |
| + t.Fatal("an unchanged edges file was rewritten") |
| + } |
| +} |
| + |
| func TestEdgesFileRawWireBodyCarriesBoxEvidence(t *testing.T) { |
| s, reg := release.NewDirStore(t.TempDir()), edgesTestRegistry(t) |
| seedRoute(t, reg, "app.example.com") |
diff --git a/internal/host/movesoffline_test.go b/internal/host/movesoffline_test.go| index dcc39c6f8..1eb64817d 100644 |
| --- a/internal/host/movesoffline_test.go |
| +++ b/internal/host/movesoffline_test.go |
| @@ -1,7 +1,6 @@ |
| package host |
| import ( |
| - "bytes" |
| "context" |
| "encoding/json" |
| "errors" |
| @@ -924,7 +923,7 @@ func TestOfflineCapabilityAndAnIncapableBox(t *testing.T) { |
| if armedOfflineCapable() { |
| t.Fatal("capable with no worker armed") |
| } |
| - if got := withOfflineCapability(edgesFileCapabilities(env()), armedOfflineCapable()); len(got) != 0 { |
| + if got := withOfflineCapability(edgesFileCapabilities(env()), armedOfflineCapable()); !slices.Equal(got, []string{capLifetimes1}) { |
| t.Fatalf("no worker armed: %v", got) |
| } |
| wdCap := newOffWorld(t) |
| @@ -933,11 +932,11 @@ func TestOfflineCapabilityAndAnIncapableBox(t *testing.T) { |
| if !armedOfflineCapable() { |
| t.Fatal("a capable armed worker does not publish") |
| } |
| - if got := withOfflineCapability(edgesFileCapabilities(env()), armedOfflineCapable()); !slices.Equal(got, []string{capOfflineMove1, capOfflineStage1}) { |
| + if got := withOfflineCapability(edgesFileCapabilities(env()), armedOfflineCapable()); !slices.Equal(got, []string{capLifetimes1, capOfflineMove1, capOfflineStage1}) { |
| t.Fatalf("capable box: %v", got) |
| } |
| base := edgesFileCapabilities(env("CARLOS_REGION_HA", "1")) |
| - if got := withOfflineCapability(base, true); !slices.Equal(got, []string{capRegionHA1, capOfflineMove1, capOfflineStage1}) || !slices.Equal(base, []string{capRegionHA1}) { |
| + if got := withOfflineCapability(base, true); !slices.Equal(got, []string{capLifetimes1, capRegionHA1, capOfflineMove1, capOfflineStage1}) || !slices.Equal(base, []string{capLifetimes1, capRegionHA1}) { |
| t.Fatalf("region HA too: %v (base %v)", got, base) |
| } |
| for _, kv := range [][]string{ |
| @@ -1128,18 +1127,21 @@ func TestOfflineDormantIsMain(t *testing.T) { |
| if json.Unmarshal(b1, &m1) != nil || json.Unmarshal(b2, &m2) != nil { |
| t.Fatal("edges files do not parse") |
| } |
| - if caps, _ := m2["capabilities"].([]any); len(caps) != 2 || caps[0] != capOfflineMove1 || caps[1] != capOfflineStage1 { |
| + if caps, _ := m2["capabilities"].([]any); len(caps) != 3 || caps[0] != capLifetimes1 || caps[1] != capOfflineMove1 || caps[2] != capOfflineStage1 { |
| t.Fatalf("capabilities: %v", m2["capabilities"]) |
| } |
| + // The plain file carries the lifetimes floor alone. |
| + if caps, _ := m1["capabilities"].([]any); len(caps) != 1 || caps[0] != capLifetimes1 { |
| + t.Fatalf("fixture: the plain file's capabilities are %v", m1["capabilities"]) |
| + } |
| + delete(m1, "capabilities") |
| delete(m2, "capabilities") |
| delete(m1, "generation") |
| delete(m2, "generation") |
| if !reflect.DeepEqual(m1, m2) { |
| t.Fatalf("the edges file differs beyond the capability:\n %s\n %s", b1, b2) |
| } |
| - if bytes.Contains(b1, []byte("capabilities")) { |
| - t.Fatal("fixture: the plain file has capabilities") |
| - } |
| + |
| } |
| // M1: a failed startup listing keeps every row only until the next |