amadan

carlos / platform Public

settings-3 final review: the fleet gate fails closed when the fleet registry was not read

An unset registry path, or a registry file that is not there, returned a
platform-only inventory that looked complete, so the update settings gate
opened without checking any fleet box. The inventory now says whether the
fleets were actually listed (and why not); only updatesEnforced reads it,
and blocks with "the fleet registry could not be read (...)", which the
doors log. The GeoIP verdict is unchanged. Tests cover unset and missing
registries, a fleet box with no, old or stale reports, and an unreadable
heartbeat, each refused through both enabling doors with clearing still
allowed.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 7984835813373144d36ed0b70ca77ef4c1a4341e parent 4b7da83
3 files changed, +218 −4
  • internal/console/apisystemgeoip.go +14 −0
  • internal/console/updatesgate.go +13 −4
  • internal/console/updatesgate_test.go +191 −0
diff --git a/internal/console/apisystemgeoip.go b/internal/console/apisystemgeoip.go
index 51ca9fb28..bec97a9a6 100644
--- a/internal/console/apisystemgeoip.go
+++ b/internal/console/apisystemgeoip.go
@@ -203,11 +203,20 @@ func (k geoipBoxKey) String() string {
// be read: informational only, because an expected box whose record was
// unreadable is missing from boxes and blocks as "no heartbeat" on its own
// (codex R9 on R1-16).
+//
+// fleetsListed is true only once the fleet registry was opened and its
+// fleets listed; fleetsUnread says why not, for the operator log. The GeoIP
+// verdict never reads it: it treats an unset or absent registry as a
+// platform-only deployment, as it always has. The update settings gate
+// (updatesEnforced) does read it, because a fleet list it never saw is not
+// evidence that there are no fleet boxes to wait for (spec §4.4).
type geoipInventoryRead struct {
expected map[geoipBoxKey]bool
boxes map[geoipBoxKey]status.Box
expectedComplete bool
heartbeatsComplete bool
+ fleetsListed bool
+ fleetsUnread string
}
// geoipWantState is what the published pointer asks of every mirror.
@@ -256,21 +265,26 @@ func (s *Server) geoipInventory(ctx context.Context) (geoipInventoryRead, error)
return inv, err
}
if s.cfg.Registry == "" {
+ inv.fleetsUnread = "no registry path is configured"
return inv, nil
}
reg, err := s.openPassRegistry()
switch {
case errors.Is(err, os.ErrNotExist):
+ inv.fleetsUnread = err.Error()
return inv, nil
case err != nil:
inv.expectedComplete = false
+ inv.fleetsUnread = err.Error()
return inv, nil
}
fleets, err := reg.ListFleets()
if err != nil {
inv.expectedComplete = false
+ inv.fleetsUnread = "listing fleets: " + err.Error()
return inv, nil
}
+ inv.fleetsListed = true
for _, f := range fleets {
fbs, err := reg.ListFleetBoxes(f.Name)
if err != nil {
diff --git a/internal/console/updatesgate.go b/internal/console/updatesgate.go
index a91a0bc50..cba70c470 100644
--- a/internal/console/updatesgate.go
+++ b/internal/console/updatesgate.go
@@ -15,9 +15,10 @@ import (
// build. Expected boxes are the edges records and active fleet members, as for
// the GeoIP rollout verdict (geoipInventory); a box that is expected and has
// no fresh heartbeat blocks, and so does an incomplete read. Unlike the GeoIP
-// verdict, an incomplete heartbeat read blocks here too, and so does an
-// inventory that found no box at all: a gate that has seen nothing cannot
-// vouch for every box.
+// verdict, an incomplete heartbeat read blocks here too, and so do an
+// inventory that found no box at all and a fleet registry that was never
+// read (unset, absent, or failing): a gate that has not seen every box
+// cannot vouch for every box.
//
// The blocking lines name platform box labels and fleet names. They are
// operator detail, logged by the doors and never shown to app members.
@@ -50,7 +51,15 @@ func (s *Server) updatesEnforced(ctx context.Context) (ok bool, blocking []strin
if len(keys) == 0 {
blocking = append(blocking, "no box was found to report a binary")
}
- if !inv.expectedComplete || !inv.heartbeatsComplete {
+ // An unset or absent registry is a fleet list nobody read, not an empty
+ // one (Astra r1 #1): the GeoIP verdict may read it as platform-only, but
+ // this gate cannot vouch for fleet boxes it never listed. A registry that
+ // failed to open or list also leaves expectedComplete false; this one
+ // line, with the reason, covers both.
+ if !inv.fleetsListed {
+ blocking = append(blocking, "the fleet registry could not be read ("+inv.fleetsUnread+"), so no fleet box was checked")
+ }
+ if (inv.fleetsListed && !inv.expectedComplete) || !inv.heartbeatsComplete {
blocking = append(blocking, "the list of boxes could not be read in full")
}
sort.Strings(blocking)
diff --git a/internal/console/updatesgate_test.go b/internal/console/updatesgate_test.go
index 0fd2939e4..c4630efd5 100644
--- a/internal/console/updatesgate_test.go
+++ b/internal/console/updatesgate_test.go
@@ -2,11 +2,17 @@ package console
import (
"context"
+ "net/http"
+ "path/filepath"
"strings"
"testing"
"time"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
"github.com/carlosframework/platform/internal/status"
+ "github.com/carlosframework/platform/internal/updatewords"
)
// seedSettingsBox writes label's heartbeat at at carrying m as its private
@@ -66,6 +72,191 @@ func TestUpdatesEnforcedNeedsSomeBox(t *testing.T) {
}
}
+// seedSettingsFleet registers fleet fx with one active box, bx-1, in the
+// fixture's registry, and (when at is not zero) writes bx-1's heartbeat
+// under the fleet's prefix carrying m.
+func seedSettingsFleet(t *testing.T, fx *consoleFixture, at time.Time, m *status.BoxMetrics) {
+ t.Helper()
+ reg, err := registry.Open(fx.regPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := reg.CreateFleet(registry.Fleet{Name: "fx", Account: fx.sqid, Bucket: "fx-data"}); err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.AddFleetBox(registry.FleetBox{Fleet: "fx", Label: "bx-1", TokenHash: strings.Repeat("a", 64)}); err != nil {
+ t.Fatal(err)
+ }
+ if at.IsZero() {
+ return
+ }
+ b := status.CollectBox(nil, status.Probes{Release: "v1"}, at)
+ b.Label = "bx-1"
+ b.Metrics = m
+ if err := status.WriteBox(context.Background(), release.NewPrefixStore(fx.rel, "fleets/fx/"), b); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// Astra r1 #1: an unread fleet inventory is not an empty one. A console
+// whose registry path is unset, or names no file, has not listed the
+// fleets, so it cannot vouch for their boxes and the gate blocks, while the
+// GeoIP verdict's reading of the same inventory is unchanged (its
+// expected list stays complete, as before).
+func TestUpdatesEnforcedNeedsTheFleetRegistry(t *testing.T) {
+ for name, path := range map[string]func(t *testing.T) string{
+ "unset": func(*testing.T) string { return "" },
+ "missing": func(t *testing.T) string { return filepath.Join(t.TempDir(), "absent-registry.db") },
+ } {
+ t.Run(name, func(t *testing.T) {
+ fx := newConsoleFixture(t)
+ seedEdgesRecord(t, fx, "ie-1")
+ seedSettingsBox(t, fx, "ie-1", time.Now(), &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol})
+ if ok, blocking, err := fx.srv.updatesEnforced(context.Background()); !ok || err != nil {
+ t.Fatalf("with the registry read: %v %q %v", ok, blocking, err)
+ }
+ fx.srv.cfg.Registry = path(t)
+ ok, blocking, err := fx.srv.updatesEnforced(context.Background())
+ if err != nil || ok || len(blocking) != 1 || !strings.Contains(blocking[0], "the fleet registry could not be read") {
+ t.Fatalf("registry %s: %v %q %v", name, ok, blocking, err)
+ }
+ inv, err := fx.srv.geoipInventory(context.Background())
+ if err != nil || !inv.expectedComplete || !inv.heartbeatsComplete {
+ t.Fatalf("the GeoIP reading of an absent registry changed: %+v %v", inv, err)
+ }
+ })
+ }
+}
+
+// Task 5's deferred gap: an active fleet box is expected, keyed by its
+// fleet, and blocks with no report, an old binary or a stale report; once
+// it reports the protocol the gate opens.
+func TestUpdatesEnforcedNeedsEveryFleetBox(t *testing.T) {
+ now := time.Now()
+ for name, c := range map[string]struct {
+ at time.Time
+ m *status.BoxMetrics
+ want string
+ }{
+ "no report": {want: "fx/bx-1: no heartbeat"},
+ "an old binary": {at: now, m: &status.BoxMetrics{}, want: "fx/bx-1: runs a binary that does not apply update settings"},
+ "a stale report": {at: now.Add(-3 * status.SnapshotPeriod), m: &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol},
+ want: "fx/bx-1: no heartbeat since"},
+ "the protocol": {at: now, m: &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol}},
+ } {
+ t.Run(name, func(t *testing.T) {
+ fx := newConsoleFixture(t)
+ seedEdgesRecord(t, fx, "ie-1")
+ seedSettingsBox(t, fx, "ie-1", now, &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol})
+ seedSettingsFleet(t, fx, c.at, c.m)
+ ok, blocking, err := fx.srv.updatesEnforced(context.Background())
+ if c.want == "" {
+ if !ok || err != nil {
+ t.Fatalf("a fleet box that applies settings: %v %q %v", ok, blocking, err)
+ }
+ return
+ }
+ if err != nil || ok || len(blocking) != 1 || !strings.HasPrefix(blocking[0], c.want) {
+ t.Fatalf("want %q blocking: %v %q %v", c.want, ok, blocking, err)
+ }
+ })
+ }
+}
+
+// Task 5's deferred gap: a heartbeat record that cannot be read leaves the
+// read incomplete, and an incomplete read blocks even when every box that
+// was read applies settings.
+func TestUpdatesEnforcedBlocksAnIncompleteRead(t *testing.T) {
+ fx := newConsoleFixture(t)
+ seedEdgesRecord(t, fx, "ie-1")
+ seedSettingsBox(t, fx, "ie-1", time.Now(), &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol})
+ if _, err := fx.rel.Put(context.Background(), "status/boxes/zz-1.json", []byte("{not json"), ""); err != nil {
+ t.Fatal(err)
+ }
+ ok, blocking, err := fx.srv.updatesEnforced(context.Background())
+ if err != nil || ok || len(blocking) != 1 || !strings.Contains(blocking[0], "could not be read in full") {
+ t.Fatalf("an unreadable heartbeat: %v %q %v", ok, blocking, err)
+ }
+}
+
+// Astra r1 #1 through the doors: whenever the gate cannot vouch for every
+// box (no registry path, a registry file that is not there, a fleet box with
+// no report, an old binary or a stale report, an unreadable heartbeat), both
+// enabling doors refuse with the not-enforced 409 and write nothing, and
+// clearing through both still answers 200 in that same state.
+func TestEnableDoorsFailClosedOnAnUncheckedBox(t *testing.T) {
+ now := time.Now()
+ notEnforced := strings.SplitN(updatewords.Text("settings.console.not-enforced"), "%d", 2)[0]
+ for name, breakIt := range map[string]func(t *testing.T, fx *updatesFx){
+ "registry unset": func(t *testing.T, fx *updatesFx) { fx.srv.cfg.Registry = "" },
+ "registry missing": func(t *testing.T, fx *updatesFx) {
+ fx.srv.cfg.Registry = filepath.Join(t.TempDir(), "absent-registry.db")
+ },
+ "fleet box with no report": func(t *testing.T, fx *updatesFx) {
+ seedSettingsFleet(t, fx.consoleFixture, time.Time{}, nil)
+ },
+ "fleet box with an old binary": func(t *testing.T, fx *updatesFx) {
+ seedSettingsFleet(t, fx.consoleFixture, now, &status.BoxMetrics{})
+ },
+ "fleet box with a stale report": func(t *testing.T, fx *updatesFx) {
+ seedSettingsFleet(t, fx.consoleFixture, now.Add(-3*status.SnapshotPeriod), &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol})
+ },
+ "an unreadable heartbeat": func(t *testing.T, fx *updatesFx) {
+ if _, err := fx.rel.Put(context.Background(), "status/boxes/zz-1.json", []byte("{not json"), ""); err != nil {
+ t.Fatal(err)
+ }
+ },
+ } {
+ t.Run(name, func(t *testing.T) {
+ fx := updatesFixture(t)
+ fx.boxesEnforce(true)
+ two := 2
+ fx.putRecord(provision.Record{Schema: 1, Host: "a.example.com", App: "km", Account: "bab", Channel: "stable", Pin: "v1"})
+ fx.setManifest(provision.Manifest{Schema: 1, App: "km", Domain: "example.com", Channel: "stable",
+ Updates: &provision.UpdatePolicy{DefaultMajor: &two}})
+ breakIt(t, fx)
+
+ st, out := sendJSON(t, http.MethodPut, fx.ts.URL+updURL, map[string]string{"updates": "manual"}, fx.member)
+ if st != http.StatusConflict || !strings.Contains(out, notEnforced) {
+ t.Errorf("instance door, manual: %d %s", st, out)
+ }
+ if rec := fx.record("a.example.com"); rec.Updates != "" {
+ t.Errorf("a refused instance enable wrote the record: %+v", rec)
+ }
+ st, out = sendJSON(t, http.MethodPut, fx.ts.URL+policyURL, map[string]any{"default_updates": "manual"}, fx.member)
+ if st != http.StatusConflict || !strings.Contains(out, notEnforced) {
+ t.Errorf("policy door, default manual: %d %s", st, out)
+ }
+ if p := fx.manifest().Updates; p == nil || p.DefaultUpdates != "" {
+ t.Errorf("a refused policy enable wrote the policy: %+v", p)
+ }
+
+ st, out = sendJSON(t, http.MethodPut, fx.ts.URL+updURL, map[string]any{"pin": ""}, fx.member)
+ if rec := fx.record("a.example.com"); st != http.StatusOK || rec.Pin != "" {
+ t.Fatalf("instance door, clearing a pin: %d %s, record %+v", st, out, rec)
+ }
+ st, out = sendJSON(t, http.MethodPut, fx.ts.URL+policyURL, map[string]any{"default_major": nil}, fx.member)
+ if p := fx.manifest().Updates; st != http.StatusOK || (p != nil && p.DefaultMajor != nil) {
+ t.Fatalf("policy door, clearing the default major: %d %s, policy %+v", st, out, p)
+ }
+ })
+ }
+}
+
+// The control for the test above: with the registry read and the fleet box
+// reporting the protocol, both doors enable.
+func TestEnableDoorsOpenWhenEveryFleetBoxEnforces(t *testing.T) {
+ fx := updatesFixture(t)
+ fx.boxesEnforce(true)
+ seedSettingsFleet(t, fx.consoleFixture, time.Now(), &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol})
+ if st, out := sendJSON(t, http.MethodPut, fx.ts.URL+updURL, map[string]string{"updates": "manual"}, fx.member); st != http.StatusOK {
+ t.Fatalf("instance door: %d %s", st, out)
+ }
+ if st, out := sendJSON(t, http.MethodPut, fx.ts.URL+policyURL, map[string]any{"default_updates": "manual"}, fx.member); st != http.StatusOK {
+ t.Fatalf("policy door: %d %s", st, out)
+ }
+}
+
// A heartbeat with no metrics block is a binary that does not say it
// enforces; an unexpected box that has gone stale is retired and does not
// block; an unexpected fresh box with an old binary does.