amadan
console: the box floor keeps seedfloor's guards: a label two reports claim refuses, a future-dated report is not fresh, and the watch's edges read is bounded
The status-report join by label refuses 409 "could not confirm box <label>. Try again in a minute." (seedfloor.go's wording for an ambiguous label) when two reports share the box's label. A report is fresh while it is at most ten minutes old and dated no more than a minute ahead (offlineFresh's skew), judged against the wall clock; a future-dated one answers the existing "has not reported in 10 minutes". lifetimesFloorWatch reads the edges files under instanceStoreTimeout, so a hung read cannot stall the cutover watch.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
3 files changed,
+58
−13
internal/console/lifetimes_cutover.go+1 −1internal/console/lifetimes_floor.go+36 −6internal/console/lifetimes_floor_test.go+21 −6
diff --git a/internal/console/lifetimes_cutover.go b/internal/console/lifetimes_cutover.go| index 8feb44de0..db4af0ac1 100644 |
| --- a/internal/console/lifetimes_cutover.go |
| +++ b/internal/console/lifetimes_cutover.go |
| @@ -530,7 +530,7 @@ func (s *Server) handleAPILifetimesCutover(w http.ResponseWriter, r *http.Reques |
| // written; once it is, the door answers the status. |
| if st := g.current(); st == cutoverBefore || st == cutoverUnknown { |
| fctx, fcancel := context.WithTimeout(r.Context(), instanceStoreTimeout) |
| - ref := s.lifetimesBoxFloor(fctx, now) |
| + ref := s.lifetimesBoxFloor(fctx) |
| fcancel() |
| if ref != nil { |
| jsonError(w, ref.Status, ref.Msg) |
diff --git a/internal/console/lifetimes_floor.go b/internal/console/lifetimes_floor.go| index b1bed2ade..f59ac1dd4 100644 |
| --- a/internal/console/lifetimes_floor.go |
| +++ b/internal/console/lifetimes_floor.go |
| @@ -49,6 +49,12 @@ func msgLifetimesBoxSilent(box string) string { |
| return fmt.Sprintf("box %s has not reported in 10 minutes", box) |
| } |
| +// msgLifetimesBoxAmbiguous is seedfloor.go's refusal for a label two |
| +// status reports claim, worded for one box. |
| +func msgLifetimesBoxAmbiguous(box string) string { |
| + return fmt.Sprintf("could not confirm box %s. Try again in a minute.", box) |
| +} |
| + |
| func pageLifetimesRegressed(box string) string { |
| return fmt.Sprintf("box %s regressed below the lifetimes floor", box) |
| } |
| @@ -61,10 +67,15 @@ type lifetimesFloorState struct { |
| } |
| // lifetimesBoxFloor checks every edges file of the deployment and its box's |
| -// status report at now. nil is accepted; otherwise the refusal to answer. |
| -// The status report is joined by its label to the file's box, the join every |
| -// fleet reader makes (status reports carry no other box identity). |
| -func (s *Server) lifetimesBoxFloor(ctx context.Context, now time.Time) *doorRefusal { |
| +// status report against the wall clock. nil is accepted; otherwise the |
| +// refusal to answer. The status report is joined by its label to the file's |
| +// box, the join every fleet reader makes (status reports carry no other box |
| +// identity), with seedfloor.go's guards: a label two reports claim is |
| +// ambiguous and refused, and a report is fresh only while it is at most |
| +// lifetimesBoxFresh old and not dated more than a minute ahead (the skew |
| +// offlineFresh allows), so a future-dated report does not pass. |
| +func (s *Server) lifetimesBoxFloor(ctx context.Context) *doorRefusal { |
| + now := time.Now() |
| snap, err := readEdgesSnapshot(ctx, s.cfg.Releases) |
| if err != nil || !snap.Complete { |
| log.Printf("console: lifetimes: the box floor: the edges files are not complete evidence (error %v)", err) |
| @@ -82,18 +93,34 @@ func (s *Server) lifetimesBoxFloor(ctx context.Context, now time.Time) *doorRefu |
| log.Printf("console: lifetimes: the box floor: the status reports are not complete evidence (complete %v, error %v)", complete, err) |
| } |
| reported := map[string]time.Time{} |
| + dup := map[string]bool{} |
| for _, b := range boxes { |
| + if _, seen := reported[b.Label]; seen { |
| + dup[b.Label] = true |
| + } |
| reported[b.Label] = b.GeneratedAt |
| } |
| for _, f := range files { |
| at, ok := reported[f.Box] |
| - if !ok || at.IsZero() || now.Sub(at) > lifetimesBoxFresh { |
| + switch { |
| + case ok && dup[f.Box]: |
| + log.Printf("console: lifetimes: the box floor: box %q has more than one status report with its label", f.Box) |
| + return &doorRefusal{http.StatusConflict, msgLifetimesBoxAmbiguous(f.Box)} |
| + case !ok || !lifetimesBoxReportFresh(at, now): |
| return &doorRefusal{http.StatusConflict, msgLifetimesBoxSilent(f.Box)} |
| } |
| } |
| return nil |
| } |
| +// lifetimesBoxReportFresh reports whether a status report generated at is |
| +// at most lifetimesBoxFresh old at now and not dated more than a minute |
| +// ahead of it. |
| +func lifetimesBoxReportFresh(at, now time.Time) bool { |
| + age := now.Sub(at) |
| + return !at.IsZero() && age <= lifetimesBoxFresh && age >= -time.Minute |
| +} |
| + |
| // lifetimesFloorWatch is the floor's check after the cutover, from the |
| // cutover watch: at most every lifetimesFloorEvery it reads the edges files, |
| // and pages once per lifetimesFloorPageEvery per box whose file lacks the |
| @@ -106,7 +133,10 @@ func (s *Server) lifetimesFloorWatch(ctx context.Context, now time.Time) { |
| } |
| s.floorReadAt = now |
| s.floorMu.Unlock() |
| - snap, err := readEdgesSnapshot(ctx, s.cfg.Releases) |
| + // Bounded, so a hung edges read cannot stall the cutover watch. |
| + rctx, cancel := context.WithTimeout(ctx, instanceStoreTimeout) |
| + snap, err := readEdgesSnapshot(rctx, s.cfg.Releases) |
| + cancel() |
| if err != nil { |
| log.Printf("console: lifetimes: the box floor watch: %v", err) |
| return |
diff --git a/internal/console/lifetimes_floor_test.go b/internal/console/lifetimes_floor_test.go| index a40acd1db..f6b6edaf9 100644 |
| --- a/internal/console/lifetimes_floor_test.go |
| +++ b/internal/console/lifetimes_floor_test.go |
| @@ -29,9 +29,15 @@ func writeEdges(t *testing.T, st release.Store, box string, caps ...string) { |
| } |
| func writeBoxStatus(t *testing.T, st release.Store, label string, at time.Time) { |
| + t.Helper() |
| + writeBoxStatusAt(t, st, label, label, at) |
| +} |
| + |
| +// writeBoxStatusAt writes a status report for label under status/boxes/<key>.json. |
| +func writeBoxStatusAt(t *testing.T, st release.Store, key, label string, at time.Time) { |
| t.Helper() |
| body, _ := json.Marshal(map[string]any{"generated_at": at, "label": label, "routes": map[string]any{}, "replication": map[string]any{}}) |
| - if _, err := st.Put(context.Background(), "status/boxes/"+label+".json", body, ""); err != nil { |
| + if _, err := st.Put(context.Background(), "status/boxes/"+key+".json", body, ""); err != nil { |
| t.Fatal(err) |
| } |
| } |
| @@ -46,19 +52,28 @@ func TestCutoverRefusesBoxesBelowTheFloor(t *testing.T) { |
| {"a box without the capability", func(t *testing.T, l lifetimesFx) { |
| writeEdges(t, l.fx.rel, "annie", capLifetimes1) |
| writeEdges(t, l.fx.rel, "bert") |
| - writeBoxStatus(t, l.fx.rel, "annie", l.clock.Now()) |
| - writeBoxStatus(t, l.fx.rel, "bert", l.clock.Now()) |
| + writeBoxStatus(t, l.fx.rel, "annie", time.Now()) |
| + writeBoxStatus(t, l.fx.rel, "bert", time.Now()) |
| }, http.StatusConflict, "box bert does not publish lifetimes-1; every box must run this release before the cutover"}, |
| {"a stale status report", func(t *testing.T, l lifetimesFx) { |
| writeEdges(t, l.fx.rel, "annie", capLifetimes1) |
| - writeBoxStatus(t, l.fx.rel, "annie", l.clock.Now().Add(-11*time.Minute)) |
| + writeBoxStatus(t, l.fx.rel, "annie", time.Now().Add(-11*time.Minute)) |
| }, http.StatusConflict, "box annie has not reported in 10 minutes"}, |
| + {"a future-dated status report", func(t *testing.T, l lifetimesFx) { |
| + writeEdges(t, l.fx.rel, "annie", capLifetimes1) |
| + writeBoxStatus(t, l.fx.rel, "annie", time.Now().Add(11*time.Minute)) |
| + }, http.StatusConflict, "box annie has not reported in 10 minutes"}, |
| + {"two status reports with one label", func(t *testing.T, l lifetimesFx) { |
| + writeEdges(t, l.fx.rel, "annie", capLifetimes1) |
| + writeBoxStatus(t, l.fx.rel, "annie", time.Now()) |
| + writeBoxStatusAt(t, l.fx.rel, "annie-clone", "annie", time.Now()) |
| + }, http.StatusConflict, "could not confirm box annie. Try again in a minute."}, |
| {"a missing status report", func(t *testing.T, l lifetimesFx) { |
| writeEdges(t, l.fx.rel, "annie", capLifetimes1) |
| }, http.StatusConflict, "box annie has not reported in 10 minutes"}, |
| {"an old edges file and a fresh report", func(t *testing.T, l lifetimesFx) { |
| writeEdges(t, l.fx.rel, "annie", capLifetimes1, "region-ha-1") |
| - writeBoxStatus(t, l.fx.rel, "annie", l.clock.Now().Add(-time.Minute)) |
| + writeBoxStatus(t, l.fx.rel, "annie", time.Now().Add(-time.Minute)) |
| }, http.StatusOK, ""}, |
| } |
| for _, c := range cases { |
| @@ -78,7 +93,7 @@ func TestCutoverRefusesBoxesBelowTheFloor(t *testing.T) { |
| t.Run("a file losing the capability after the cutover pages hourly", func(t *testing.T) { |
| l := newLifetimesFx(t) |
| writeEdges(t, l.fx.rel, "annie", capLifetimes1) |
| - writeBoxStatus(t, l.fx.rel, "annie", l.clock.Now()) |
| + writeBoxStatus(t, l.fx.rel, "annie", time.Now()) |
| turnLifetimesOn(t, l.fx.srv, l.clock) |
| var pages []string |
| var mu sync.Mutex |