| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "net/http" |
| 6 | "path/filepath" |
| 7 | "strings" |
| 8 | "testing" |
| 9 | "time" |
| 10 | |
| 11 | "github.com/carlosframework/platform/internal/provision" |
| 12 | "github.com/carlosframework/platform/internal/registry" |
| 13 | "github.com/carlosframework/platform/internal/release" |
| 14 | "github.com/carlosframework/platform/internal/status" |
| 15 | "github.com/carlosframework/platform/internal/updatewords" |
| 16 | ) |
| 17 | |
| 18 | // seedSettingsBox writes label's heartbeat at at carrying m as its private |
| 19 | // metrics block (nil writes a heartbeat with no metrics at all). |
| 20 | func seedSettingsBox(t *testing.T, fx *consoleFixture, label string, at time.Time, m *status.BoxMetrics) { |
| 21 | t.Helper() |
| 22 | b := status.CollectBox(nil, status.Probes{Release: "v1"}, at) |
| 23 | b.Label = label |
| 24 | b.Metrics = m |
| 25 | if err := status.WriteBox(context.Background(), fx.rel, b); err != nil { |
| 26 | t.Fatal(err) |
| 27 | } |
| 28 | } |
| 29 | |
| 30 | func TestUpdatesEnforcedNeedsEveryBox(t *testing.T) { |
| 31 | fx := newConsoleFixture(t) |
| 32 | seedEdgesRecord(t, fx, "ie-1") |
| 33 | seedEdgesRecord(t, fx, "uk-1") |
| 34 | now := time.Now() |
| 35 | seedSettingsBox(t, fx, "ie-1", now, &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol}) |
| 36 | seedSettingsBox(t, fx, "uk-1", now, &status.BoxMetrics{}) |
| 37 | ok, blocking, err := fx.srv.updatesEnforced(context.Background()) |
| 38 | if err != nil || ok || len(blocking) != 1 || !strings.Contains(blocking[0], "uk-1") { |
| 39 | t.Fatalf("one old binary: %v %q %v", ok, blocking, err) |
| 40 | } |
| 41 | seedSettingsBox(t, fx, "uk-1", now, &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol}) |
| 42 | if ok, _, err := fx.srv.updatesEnforced(context.Background()); !ok || err != nil { |
| 43 | t.Fatalf("every box enforces: %v %v", ok, err) |
| 44 | } |
| 45 | } |
| 46 | |
| 47 | // Review Focus 4. |
| 48 | func TestUpdatesEnforcedNamesAStaleBox(t *testing.T) { |
| 49 | fx := newConsoleFixture(t) |
| 50 | seedEdgesRecord(t, fx, "ie-1") |
| 51 | seedSettingsBox(t, fx, "ie-1", time.Now().Add(-3*status.SnapshotPeriod), &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol}) |
| 52 | ok, blocking, _ := fx.srv.updatesEnforced(context.Background()) |
| 53 | if ok || len(blocking) != 1 || !strings.Contains(blocking[0], "ie-1") { |
| 54 | t.Fatalf("a stale box: %v %q", ok, blocking) |
| 55 | } |
| 56 | } |
| 57 | |
| 58 | func TestUpdatesEnforcedNeedsAHeartbeat(t *testing.T) { |
| 59 | fx := newConsoleFixture(t) |
| 60 | seedEdgesRecord(t, fx, "ie-1") |
| 61 | if ok, blocking, _ := fx.srv.updatesEnforced(context.Background()); ok || len(blocking) != 1 { |
| 62 | t.Fatalf("an expected box with no heartbeat: %v %q", ok, blocking) |
| 63 | } |
| 64 | } |
| 65 | |
| 66 | // An empty inventory has seen no box at all, so it cannot vouch that every |
| 67 | // box enforces: it blocks rather than reading as "nothing to wait on". |
| 68 | func TestUpdatesEnforcedNeedsSomeBox(t *testing.T) { |
| 69 | fx := newConsoleFixture(t) |
| 70 | if ok, blocking, err := fx.srv.updatesEnforced(context.Background()); ok || err != nil || len(blocking) != 1 { |
| 71 | t.Fatalf("no box known: %v %q %v", ok, blocking, err) |
| 72 | } |
| 73 | } |
| 74 | |
| 75 | // seedSettingsFleet registers fleet fx with one active box, bx-1, in the |
| 76 | // fixture's registry, and (when at is not zero) writes bx-1's heartbeat |
| 77 | // under the fleet's prefix carrying m. |
| 78 | func seedSettingsFleet(t *testing.T, fx *consoleFixture, at time.Time, m *status.BoxMetrics) { |
| 79 | t.Helper() |
| 80 | reg, err := registry.Open(fx.regPath) |
| 81 | if err != nil { |
| 82 | t.Fatal(err) |
| 83 | } |
| 84 | if _, err := reg.CreateFleet(registry.Fleet{Name: "fx", Account: fx.sqid, Bucket: "fx-data"}); err != nil { |
| 85 | t.Fatal(err) |
| 86 | } |
| 87 | if err := reg.AddFleetBox(registry.FleetBox{Fleet: "fx", Label: "bx-1", TokenHash: strings.Repeat("a", 64)}); err != nil { |
| 88 | t.Fatal(err) |
| 89 | } |
| 90 | if at.IsZero() { |
| 91 | return |
| 92 | } |
| 93 | b := status.CollectBox(nil, status.Probes{Release: "v1"}, at) |
| 94 | b.Label = "bx-1" |
| 95 | b.Metrics = m |
| 96 | if err := status.WriteBox(context.Background(), release.NewPrefixStore(fx.rel, "fleets/fx/"), b); err != nil { |
| 97 | t.Fatal(err) |
| 98 | } |
| 99 | } |
| 100 | |
| 101 | // Astra r1 #1: an unread fleet inventory is not an empty one. A console |
| 102 | // whose registry path is unset, or names no file, has not listed the |
| 103 | // fleets, so it cannot vouch for their boxes and the gate blocks, while the |
| 104 | // GeoIP verdict's reading of the same inventory is unchanged (its |
| 105 | // expected list stays complete, as before). |
| 106 | func TestUpdatesEnforcedNeedsTheFleetRegistry(t *testing.T) { |
| 107 | for name, path := range map[string]func(t *testing.T) string{ |
| 108 | "unset": func(*testing.T) string { return "" }, |
| 109 | "missing": func(t *testing.T) string { return filepath.Join(t.TempDir(), "absent-registry.db") }, |
| 110 | } { |
| 111 | t.Run(name, func(t *testing.T) { |
| 112 | fx := newConsoleFixture(t) |
| 113 | seedEdgesRecord(t, fx, "ie-1") |
| 114 | seedSettingsBox(t, fx, "ie-1", time.Now(), &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol}) |
| 115 | if ok, blocking, err := fx.srv.updatesEnforced(context.Background()); !ok || err != nil { |
| 116 | t.Fatalf("with the registry read: %v %q %v", ok, blocking, err) |
| 117 | } |
| 118 | fx.srv.cfg.Registry = path(t) |
| 119 | ok, blocking, err := fx.srv.updatesEnforced(context.Background()) |
| 120 | if err != nil || ok || len(blocking) != 1 || !strings.Contains(blocking[0], "the fleet registry could not be read") { |
| 121 | t.Fatalf("registry %s: %v %q %v", name, ok, blocking, err) |
| 122 | } |
| 123 | inv, err := fx.srv.geoipInventory(context.Background()) |
| 124 | if err != nil || !inv.expectedComplete || !inv.heartbeatsComplete { |
| 125 | t.Fatalf("the GeoIP reading of an absent registry changed: %+v %v", inv, err) |
| 126 | } |
| 127 | }) |
| 128 | } |
| 129 | } |
| 130 | |
| 131 | // Task 5's deferred gap: an active fleet box is expected, keyed by its |
| 132 | // fleet, and blocks with no report, an old binary or a stale report; once |
| 133 | // it reports the protocol the gate opens. |
| 134 | func TestUpdatesEnforcedNeedsEveryFleetBox(t *testing.T) { |
| 135 | now := time.Now() |
| 136 | for name, c := range map[string]struct { |
| 137 | at time.Time |
| 138 | m *status.BoxMetrics |
| 139 | want string |
| 140 | }{ |
| 141 | "no report": {want: "fx/bx-1: no heartbeat"}, |
| 142 | "an old binary": {at: now, m: &status.BoxMetrics{}, want: "fx/bx-1: runs a binary that does not apply update settings"}, |
| 143 | "a stale report": {at: now.Add(-3 * status.SnapshotPeriod), m: &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol}, |
| 144 | want: "fx/bx-1: no heartbeat since"}, |
| 145 | "the protocol": {at: now, m: &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol}}, |
| 146 | } { |
| 147 | t.Run(name, func(t *testing.T) { |
| 148 | fx := newConsoleFixture(t) |
| 149 | seedEdgesRecord(t, fx, "ie-1") |
| 150 | seedSettingsBox(t, fx, "ie-1", now, &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol}) |
| 151 | seedSettingsFleet(t, fx, c.at, c.m) |
| 152 | ok, blocking, err := fx.srv.updatesEnforced(context.Background()) |
| 153 | if c.want == "" { |
| 154 | if !ok || err != nil { |
| 155 | t.Fatalf("a fleet box that applies settings: %v %q %v", ok, blocking, err) |
| 156 | } |
| 157 | return |
| 158 | } |
| 159 | if err != nil || ok || len(blocking) != 1 || !strings.HasPrefix(blocking[0], c.want) { |
| 160 | t.Fatalf("want %q blocking: %v %q %v", c.want, ok, blocking, err) |
| 161 | } |
| 162 | }) |
| 163 | } |
| 164 | } |
| 165 | |
| 166 | // Task 5's deferred gap: a heartbeat record that cannot be read leaves the |
| 167 | // read incomplete, and an incomplete read blocks even when every box that |
| 168 | // was read applies settings. |
| 169 | func TestUpdatesEnforcedBlocksAnIncompleteRead(t *testing.T) { |
| 170 | fx := newConsoleFixture(t) |
| 171 | seedEdgesRecord(t, fx, "ie-1") |
| 172 | seedSettingsBox(t, fx, "ie-1", time.Now(), &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol}) |
| 173 | if _, err := fx.rel.Put(context.Background(), "status/boxes/zz-1.json", []byte("{not json"), ""); err != nil { |
| 174 | t.Fatal(err) |
| 175 | } |
| 176 | ok, blocking, err := fx.srv.updatesEnforced(context.Background()) |
| 177 | if err != nil || ok || len(blocking) != 1 || !strings.Contains(blocking[0], "could not be read in full") { |
| 178 | t.Fatalf("an unreadable heartbeat: %v %q %v", ok, blocking, err) |
| 179 | } |
| 180 | } |
| 181 | |
| 182 | // Astra r1 #1 through the doors: whenever the gate cannot vouch for every |
| 183 | // box (no registry path, a registry file that is not there, a fleet box with |
| 184 | // no report, an old binary or a stale report, an unreadable heartbeat), both |
| 185 | // enabling doors refuse with the not-enforced 409 and write nothing, and |
| 186 | // clearing through both still answers 200 in that same state. |
| 187 | func TestEnableDoorsFailClosedOnAnUncheckedBox(t *testing.T) { |
| 188 | now := time.Now() |
| 189 | notEnforced := strings.SplitN(updatewords.Text("settings.console.not-enforced"), "%d", 2)[0] |
| 190 | for name, breakIt := range map[string]func(t *testing.T, fx *updatesFx){ |
| 191 | "registry unset": func(t *testing.T, fx *updatesFx) { fx.srv.cfg.Registry = "" }, |
| 192 | "registry missing": func(t *testing.T, fx *updatesFx) { |
| 193 | fx.srv.cfg.Registry = filepath.Join(t.TempDir(), "absent-registry.db") |
| 194 | }, |
| 195 | "fleet box with no report": func(t *testing.T, fx *updatesFx) { |
| 196 | seedSettingsFleet(t, fx.consoleFixture, time.Time{}, nil) |
| 197 | }, |
| 198 | "fleet box with an old binary": func(t *testing.T, fx *updatesFx) { |
| 199 | seedSettingsFleet(t, fx.consoleFixture, now, &status.BoxMetrics{}) |
| 200 | }, |
| 201 | "fleet box with a stale report": func(t *testing.T, fx *updatesFx) { |
| 202 | seedSettingsFleet(t, fx.consoleFixture, now.Add(-3*status.SnapshotPeriod), &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol}) |
| 203 | }, |
| 204 | "an unreadable heartbeat": func(t *testing.T, fx *updatesFx) { |
| 205 | if _, err := fx.rel.Put(context.Background(), "status/boxes/zz-1.json", []byte("{not json"), ""); err != nil { |
| 206 | t.Fatal(err) |
| 207 | } |
| 208 | }, |
| 209 | } { |
| 210 | t.Run(name, func(t *testing.T) { |
| 211 | fx := updatesFixture(t) |
| 212 | fx.boxesEnforce(true) |
| 213 | two := 2 |
| 214 | fx.putRecord(provision.Record{Schema: 1, Host: "a.example.com", App: "km", Account: "bab", Channel: "stable", Pin: "v1"}) |
| 215 | fx.setManifest(provision.Manifest{Schema: 1, App: "km", Domain: "example.com", Channel: "stable", |
| 216 | Updates: &provision.UpdatePolicy{DefaultMajor: &two}}) |
| 217 | breakIt(t, fx) |
| 218 | |
| 219 | st, out := sendJSON(t, http.MethodPut, fx.ts.URL+updURL, map[string]string{"updates": "manual"}, fx.member) |
| 220 | if st != http.StatusConflict || !strings.Contains(out, notEnforced) { |
| 221 | t.Errorf("instance door, manual: %d %s", st, out) |
| 222 | } |
| 223 | if rec := fx.record("a.example.com"); rec.Updates != "" { |
| 224 | t.Errorf("a refused instance enable wrote the record: %+v", rec) |
| 225 | } |
| 226 | st, out = sendJSON(t, http.MethodPut, fx.ts.URL+policyURL, map[string]any{"default_updates": "manual"}, fx.member) |
| 227 | if st != http.StatusConflict || !strings.Contains(out, notEnforced) { |
| 228 | t.Errorf("policy door, default manual: %d %s", st, out) |
| 229 | } |
| 230 | if p := fx.manifest().Updates; p == nil || p.DefaultUpdates != "" { |
| 231 | t.Errorf("a refused policy enable wrote the policy: %+v", p) |
| 232 | } |
| 233 | |
| 234 | st, out = sendJSON(t, http.MethodPut, fx.ts.URL+updURL, map[string]any{"pin": ""}, fx.member) |
| 235 | if rec := fx.record("a.example.com"); st != http.StatusOK || rec.Pin != "" { |
| 236 | t.Fatalf("instance door, clearing a pin: %d %s, record %+v", st, out, rec) |
| 237 | } |
| 238 | st, out = sendJSON(t, http.MethodPut, fx.ts.URL+policyURL, map[string]any{"default_major": nil}, fx.member) |
| 239 | if p := fx.manifest().Updates; st != http.StatusOK || (p != nil && p.DefaultMajor != nil) { |
| 240 | t.Fatalf("policy door, clearing the default major: %d %s, policy %+v", st, out, p) |
| 241 | } |
| 242 | }) |
| 243 | } |
| 244 | } |
| 245 | |
| 246 | // The control for the test above: with the registry read and the fleet box |
| 247 | // reporting the protocol, both doors enable. |
| 248 | func TestEnableDoorsOpenWhenEveryFleetBoxEnforces(t *testing.T) { |
| 249 | fx := updatesFixture(t) |
| 250 | fx.boxesEnforce(true) |
| 251 | seedSettingsFleet(t, fx.consoleFixture, time.Now(), &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol}) |
| 252 | if st, out := sendJSON(t, http.MethodPut, fx.ts.URL+updURL, map[string]string{"updates": "manual"}, fx.member); st != http.StatusOK { |
| 253 | t.Fatalf("instance door: %d %s", st, out) |
| 254 | } |
| 255 | if st, out := sendJSON(t, http.MethodPut, fx.ts.URL+policyURL, map[string]any{"default_updates": "manual"}, fx.member); st != http.StatusOK { |
| 256 | t.Fatalf("policy door: %d %s", st, out) |
| 257 | } |
| 258 | } |
| 259 | |
| 260 | // A heartbeat with no metrics block is a binary that does not say it |
| 261 | // enforces; an unexpected box that has gone stale is retired and does not |
| 262 | // block; an unexpected fresh box with an old binary does. |
| 263 | func TestUpdatesEnforcedBoxShapes(t *testing.T) { |
| 264 | fx := newConsoleFixture(t) |
| 265 | now := time.Now() |
| 266 | seedEdgesRecord(t, fx, "ie-1") |
| 267 | seedSettingsBox(t, fx, "ie-1", now, &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol}) |
| 268 | seedSettingsBox(t, fx, "old-1", now.Add(-time.Hour), &status.BoxMetrics{}) |
| 269 | if ok, blocking, err := fx.srv.updatesEnforced(context.Background()); !ok || err != nil { |
| 270 | t.Fatalf("a retired box blocked: %v %q %v", ok, blocking, err) |
| 271 | } |
| 272 | seedSettingsBox(t, fx, "new-1", now, &status.BoxMetrics{}) |
| 273 | ok, blocking, _ := fx.srv.updatesEnforced(context.Background()) |
| 274 | if ok || len(blocking) != 1 || !strings.Contains(blocking[0], "new-1") { |
| 275 | t.Fatalf("an unlisted fresh old binary: %v %q", ok, blocking) |
| 276 | } |
| 277 | seedSettingsBox(t, fx, "new-1", now, &status.BoxMetrics{UpdateSettings: status.UpdateSettingsProtocol}) |
| 278 | seedSettingsBox(t, fx, "ie-1", now, nil) |
| 279 | ok, blocking, _ = fx.srv.updatesEnforced(context.Background()) |
| 280 | if ok || len(blocking) != 1 || !strings.Contains(blocking[0], "ie-1") { |
| 281 | t.Fatalf("a heartbeat with no metrics: %v %q", ok, blocking) |
| 282 | } |
| 283 | } |
| 284 | |
amadan