| 1 | package status_test |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "reflect" |
| 10 | "strings" |
| 11 | "testing" |
| 12 | "time" |
| 13 | |
| 14 | "github.com/carlosframework/platform/internal/registry" |
| 15 | "github.com/carlosframework/platform/internal/release" |
| 16 | "github.com/carlosframework/platform/internal/status" |
| 17 | ) |
| 18 | |
| 19 | // flakyPutStore wraps a real release.Store and makes its first N Put calls |
| 20 | // return release.ErrCASConflict before delegating — a stand-in for another |
| 21 | // writer racing in between WriteBox's Get-etag and Put. Every other method |
| 22 | // (Get, List) is the embedded Store's real behavior. |
| 23 | type flakyPutStore struct { |
| 24 | release.Store |
| 25 | failCount int // Put calls left to fail before delegating |
| 26 | putCalls int |
| 27 | } |
| 28 | |
| 29 | func (f *flakyPutStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| 30 | f.putCalls++ |
| 31 | if f.putCalls <= f.failCount { |
| 32 | return "", release.ErrCASConflict |
| 33 | } |
| 34 | return f.Store.Put(ctx, key, body, ifMatch) |
| 35 | } |
| 36 | |
| 37 | func TestBandBoundaries(t *testing.T) { |
| 38 | f := func(v float64) *float64 { return &v } |
| 39 | cases := []struct { |
| 40 | frac *float64 |
| 41 | want string |
| 42 | }{ |
| 43 | {f(0.69), "plenty"}, {f(0.70), "getting-full"}, |
| 44 | {f(0.90), "getting-full"}, {f(0.91), "at-capacity"}, |
| 45 | } |
| 46 | for _, c := range cases { |
| 47 | got := status.CollectBox(nil, status.Probes{MachineID: "m", DiskFrac: c.frac}, time.Now()).Capacity |
| 48 | if got == nil || *got != c.want { |
| 49 | t.Errorf("frac %v: got %v want %s", *c.frac, got, c.want) |
| 50 | } |
| 51 | } |
| 52 | if b := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Now()); b.Capacity != nil || b.Load != nil { |
| 53 | t.Error("unknown fractions must band to null, not a word") |
| 54 | } |
| 55 | } |
| 56 | |
| 57 | func TestAwakeAsleepNullWhenUnreadable(t *testing.T) { |
| 58 | routes := []registry.Route{{Host: "a", Kind: "instance", Hibernate: true, Socket: "/x.sock"}} |
| 59 | b := status.CollectBox(routes, status.Probes{MachineID: "m"}, time.Now()) // nil Listening |
| 60 | if b.Routes.Awake != nil || b.Routes.Asleep != nil { |
| 61 | t.Error("nil socket probe must publish null, never 0") |
| 62 | } |
| 63 | } |
| 64 | |
| 65 | // TestCollectBoxSkipsPlatformMintedRoutes: a platform domain is a second |
| 66 | // NAME for a route already counted, not a second instance. Counting both |
| 67 | // would roughly double every fleet-wide instance number the moment |
| 68 | // PLATFORM_APPS_DOMAIN is set — the public status page overstating the |
| 69 | // fleet, which is exactly the kind of quiet dishonesty the page's whole |
| 70 | // design exists to prevent. |
| 71 | func TestCollectBoxSkipsPlatformMintedRoutes(t *testing.T) { |
| 72 | listening := func(string) *bool { v := true; return &v } |
| 73 | routes := []registry.Route{ |
| 74 | {Host: "kass.training", Kind: "instance", Status: "ready"}, |
| 75 | {Host: "kass.bab.oncarlos.test", Kind: "instance", Status: "ready", Platform: true}, |
| 76 | {Host: "site.custom", Kind: "static"}, |
| 77 | {Host: "site.bab.oncarlos.test", Kind: "static", Platform: true}, |
| 78 | {Host: "sleepy.custom", Kind: "instance", Hibernate: true, Socket: "/s.sock"}, |
| 79 | {Host: "sleepy.bab.oncarlos.test", Kind: "instance", Hibernate: true, Socket: "/s.sock", Platform: true}, |
| 80 | } |
| 81 | b := status.CollectBox(routes, status.Probes{MachineID: "m", Listening: listening}, time.Now()) |
| 82 | if b.Routes.Resident != 1 || b.Routes.Static != 1 || b.Routes.Hibernating != 1 { |
| 83 | t.Errorf("counts = %+v, want exactly one of each: aliases must not inflate the fleet", b.Routes) |
| 84 | } |
| 85 | if b.Routes.Awake == nil || *b.Routes.Awake != 1 { |
| 86 | t.Errorf("awake = %v, want 1 — the alias must not be counted awake twice", b.Routes.Awake) |
| 87 | } |
| 88 | } |
| 89 | |
| 90 | // TestCollectBoxSidecarNotResident: a sidecar (Kind "sidecar") is not the |
| 91 | // public fleet page's business — its existence is account-visible, not |
| 92 | // public. Before an explicit branch, it fell into "default: Resident++", |
| 93 | // silently inflating the public resident-app-instance count. It must land |
| 94 | // in neither Resident, Static nor Hibernating (even when it happens to be |
| 95 | // hibernating — a sidecar's Hibernate comes from the app manifest, so it may |
| 96 | // legitimately be either), and Awake/Asleep must not count it — its Socket |
| 97 | // is empty by construction, so probing it would be meaningless. |
| 98 | func TestCollectBoxSidecarNotResident(t *testing.T) { |
| 99 | listening := func(string) *bool { v := true; return &v } |
| 100 | routes := []registry.Route{ |
| 101 | {Host: "bot.eleven.bdm.sidecar.internal", Kind: "sidecar", Status: "ready"}, |
| 102 | {Host: "bot2.eleven.bdm.sidecar.internal", Kind: "sidecar", Status: "ready", Hibernate: true}, |
| 103 | {Host: "real.custom", Kind: "instance", Status: "ready"}, |
| 104 | } |
| 105 | b := status.CollectBox(routes, status.Probes{MachineID: "m", Listening: listening}, time.Now()) |
| 106 | if b.Routes.Resident != 1 { |
| 107 | t.Errorf("Resident = %d, want 1 — a sidecar must not be counted as a public resident app instance", b.Routes.Resident) |
| 108 | } |
| 109 | if b.Routes.Hibernating != 0 { |
| 110 | t.Errorf("Hibernating = %d, want 0 — a hibernating sidecar still belongs in its own bucket, not this one", b.Routes.Hibernating) |
| 111 | } |
| 112 | if b.Sidecars != 2 { |
| 113 | t.Errorf("Sidecars = %d, want 2 — both sidecars, hibernating or not", b.Sidecars) |
| 114 | } |
| 115 | } |
| 116 | |
| 117 | // TestSidecarsNeverReachFleetJSON is the dryrun.go pattern applied to this |
| 118 | // one field: BuildFleet's per-box copy names Box's public fields one at a |
| 119 | // time (fleet.go), and Sidecars must never be one of them — a sidecar's |
| 120 | // existence is account-visible, not public. Proven by round-tripping through |
| 121 | // the real writer/merger, not just by the FleetBox struct's shape, so a |
| 122 | // future field added to FleetBox that DOES copy it would be caught here too. |
| 123 | func TestSidecarsNeverReachFleetJSON(t *testing.T) { |
| 124 | s := teststore(t) |
| 125 | now := time.Now() |
| 126 | routes := []registry.Route{{Host: "bot.x.sidecar.internal", Kind: "sidecar", Status: "ready"}} |
| 127 | b := status.CollectBox(routes, status.Probes{MachineID: "leaky"}, now) |
| 128 | if b.Sidecars != 1 { |
| 129 | t.Fatalf("Sidecars = %d, want 1 in the private snapshot itself", b.Sidecars) |
| 130 | } |
| 131 | if err := status.WriteBox(context.Background(), s, b); err != nil { |
| 132 | t.Fatal(err) |
| 133 | } |
| 134 | f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| 135 | if err != nil { |
| 136 | t.Fatal(err) |
| 137 | } |
| 138 | j, err := json.Marshal(f) |
| 139 | if err != nil { |
| 140 | t.Fatal(err) |
| 141 | } |
| 142 | if strings.Contains(strings.ToLower(string(j)), "sidecar") { |
| 143 | t.Errorf("fleet.json mentions sidecars: %s", j) |
| 144 | } |
| 145 | } |
| 146 | |
| 147 | func TestLabelStableAndOpaque(t *testing.T) { |
| 148 | a, b := status.Label("machine-1"), status.Label("machine-1") |
| 149 | if a != b || a == "" || strings.Contains(a, "machine") { |
| 150 | t.Errorf("label %q must be stable and never contain the machine id", a) |
| 151 | } |
| 152 | } |
| 153 | |
| 154 | func TestSnapshotJSONCarriesNoHostnames(t *testing.T) { |
| 155 | routes := []registry.Route{{Host: "secret.example.com", Kind: "instance", Status: "ready"}} |
| 156 | b := status.CollectBox(routes, status.Probes{MachineID: "ip-10-0-0-7"}, time.Now()) |
| 157 | j, _ := json.Marshal(b) |
| 158 | for _, leak := range []string{"secret", "example.com", "ip-10-0-0-7"} { |
| 159 | if strings.Contains(string(j), leak) { |
| 160 | t.Errorf("snapshot leaked %q: %s", leak, j) |
| 161 | } |
| 162 | } |
| 163 | } |
| 164 | |
| 165 | // TestRouteCountsJSONKeysAreSnakeCase locks RouteCounts' wire shape to the |
| 166 | // design spec's fleet.json example (docs/superpowers/specs/2026-08-01-fleet- |
| 167 | // status-page-design.md §2: {"static":2,"resident":3,"hibernating":4, |
| 168 | // "awake":1,"asleep":3}) — lowercase snake_case, not Go's default |
| 169 | // capitalized field names. |
| 170 | func TestRouteCountsJSONKeysAreSnakeCase(t *testing.T) { |
| 171 | routes := []registry.Route{ |
| 172 | {Host: "a", Kind: "static"}, |
| 173 | {Host: "b", Kind: "instance"}, |
| 174 | {Host: "c", Kind: "instance", Hibernate: true, Socket: "/c.sock"}, |
| 175 | } |
| 176 | listening := func(string) *bool { v := true; return &v } |
| 177 | b := status.CollectBox(routes, status.Probes{MachineID: "m", Listening: listening}, time.Now()) |
| 178 | j, err := json.Marshal(b.Routes) |
| 179 | if err != nil { |
| 180 | t.Fatalf("marshal RouteCounts: %v", err) |
| 181 | } |
| 182 | for _, key := range []string{`"static"`, `"resident"`, `"hibernating"`, `"awake"`, `"asleep"`} { |
| 183 | if !strings.Contains(string(j), key) { |
| 184 | t.Errorf("RouteCounts JSON %s missing key %s", j, key) |
| 185 | } |
| 186 | } |
| 187 | for _, key := range []string{`"Static"`, `"Resident"`, `"Hibernating"`, `"Awake"`, `"Asleep"`} { |
| 188 | if strings.Contains(string(j), key) { |
| 189 | t.Errorf("RouteCounts JSON %s carries capitalized Go field name %s", j, key) |
| 190 | } |
| 191 | } |
| 192 | } |
| 193 | |
| 194 | // TestWriteBoxPutsUnderLabeledKey isn't in the plan's Step-1 listing, but |
| 195 | // WriteBox is part of the exported contract Task 3 relies on — worth locking |
| 196 | // down that it actually lands the box at the key CollectBox's label implies. |
| 197 | func TestWriteBoxPutsUnderLabeledKey(t *testing.T) { |
| 198 | dir := t.TempDir() |
| 199 | store := release.NewDirStore(dir) |
| 200 | b := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Now()) |
| 201 | |
| 202 | if err := status.WriteBox(context.Background(), store, b); err != nil { |
| 203 | t.Fatalf("WriteBox: %v", err) |
| 204 | } |
| 205 | |
| 206 | key := "status/boxes/" + b.Label + ".json" |
| 207 | got, _, err := store.Get(context.Background(), key) |
| 208 | if err != nil { |
| 209 | t.Fatalf("Get %s: %v", key, err) |
| 210 | } |
| 211 | var round status.Box |
| 212 | if err := json.Unmarshal(got, &round); err != nil { |
| 213 | t.Fatalf("unmarshal written box: %v", err) |
| 214 | } |
| 215 | if round.Label != b.Label { |
| 216 | t.Errorf("round-tripped label %q, want %q", round.Label, b.Label) |
| 217 | } |
| 218 | } |
| 219 | |
| 220 | // TestWriteBoxOverwritesOnSecondTick proves WriteBox is last-writer-wins for |
| 221 | // a given label: boxes publish every 5 minutes to the same key, so a |
| 222 | // create-only Put (which fails once the object exists) would break on the |
| 223 | // second tick. Both calls must succeed, and the second call's content must |
| 224 | // be what a reader sees afterward. |
| 225 | func TestWriteBoxOverwritesOnSecondTick(t *testing.T) { |
| 226 | dir := t.TempDir() |
| 227 | store := release.NewDirStore(dir) |
| 228 | ctx := context.Background() |
| 229 | |
| 230 | first := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Unix(1000, 0)) |
| 231 | if err := status.WriteBox(ctx, store, first); err != nil { |
| 232 | t.Fatalf("first WriteBox: %v", err) |
| 233 | } |
| 234 | |
| 235 | second := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Unix(2000, 0)) |
| 236 | if err := status.WriteBox(ctx, store, second); err != nil { |
| 237 | t.Fatalf("second WriteBox (must overwrite, not conflict): %v", err) |
| 238 | } |
| 239 | |
| 240 | key := "status/boxes/" + second.Label + ".json" |
| 241 | got, _, err := store.Get(ctx, key) |
| 242 | if err != nil { |
| 243 | t.Fatalf("Get %s: %v", key, err) |
| 244 | } |
| 245 | var round status.Box |
| 246 | if err := json.Unmarshal(got, &round); err != nil { |
| 247 | t.Fatalf("unmarshal written box: %v", err) |
| 248 | } |
| 249 | if !round.GeneratedAt.Equal(second.GeneratedAt) { |
| 250 | t.Errorf("stored box has GeneratedAt %v, want the second write's %v (second write must win)", round.GeneratedAt, second.GeneratedAt) |
| 251 | } |
| 252 | } |
| 253 | |
| 254 | // TestWriteBoxRetriesOnceOnCASConflict exercises the retry branch directly: |
| 255 | // the first Put loses the CAS race (something else wrote between WriteBox's |
| 256 | // Get and its Put), so WriteBox must re-read the etag and retry exactly |
| 257 | // once — and that retry must succeed. |
| 258 | func TestWriteBoxRetriesOnceOnCASConflict(t *testing.T) { |
| 259 | dir := t.TempDir() |
| 260 | real := release.NewDirStore(dir) |
| 261 | flaky := &flakyPutStore{Store: real, failCount: 1} |
| 262 | b := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Now()) |
| 263 | |
| 264 | if err := status.WriteBox(context.Background(), flaky, b); err != nil { |
| 265 | t.Fatalf("WriteBox with one transient conflict: %v", err) |
| 266 | } |
| 267 | if flaky.putCalls != 2 { |
| 268 | t.Errorf("Put called %d times, want exactly 2 (fail once, retry once)", flaky.putCalls) |
| 269 | } |
| 270 | |
| 271 | key := "status/boxes/" + b.Label + ".json" |
| 272 | if _, _, err := real.Get(context.Background(), key); err != nil { |
| 273 | t.Fatalf("Get %s after retry: %v", key, err) |
| 274 | } |
| 275 | } |
| 276 | |
| 277 | // ---- Task 5: raw Metrics block (private to the bucket) -------------------- |
| 278 | |
| 279 | // TestCollectBoxCarriesRawMetrics proves CollectBox attaches an unbanded |
| 280 | // Metrics block straight through from Probes when the six fields are known — |
| 281 | // the one deliberate exception to this package's band-everything rule (see |
| 282 | // snapshot.go's package doc). |
| 283 | func TestCollectBoxCarriesRawMetrics(t *testing.T) { |
| 284 | mt, ma, df := int64(4056547328), int64(2028273664), int64(9999999) |
| 285 | l1 := 0.42 |
| 286 | b := status.CollectBox(nil, status.Probes{ |
| 287 | MachineID: "m", MemTotalBytes: &mt, MemAvailBytes: &ma, Load1: &l1, DiskFreeBytes: &df, |
| 288 | Release: "abc1234", Tunnel: "connected", |
| 289 | }, time.Now()) |
| 290 | m := b.Metrics |
| 291 | if m == nil || *m.MemTotalBytes != mt || *m.Load1 != l1 || m.Release != "abc1234" || m.Tunnel != "connected" { |
| 292 | t.Fatalf("metrics block wrong: %+v", m) |
| 293 | } |
| 294 | } |
| 295 | |
| 296 | // TestCollectBoxOmitsMetricsWhenAllUnknown proves the all-unknown case |
| 297 | // really omits the block rather than attaching an empty-but-present |
| 298 | // BoxMetrics{} — the same "unknown stays absent" rule every other Box field |
| 299 | // follows, and what makes an old parser's absent-key shape and a |
| 300 | // can't-answer box's shape identical on the wire. |
| 301 | func TestCollectBoxOmitsMetricsWhenAllUnknown(t *testing.T) { |
| 302 | b := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Now()) |
| 303 | if b.Metrics != nil { |
| 304 | t.Fatalf("metrics block should be nil when nothing is known, got %+v", b.Metrics) |
| 305 | } |
| 306 | } |
| 307 | |
| 308 | // TestBoxBackCompatParse: yesterday's Box shape (no metrics key) must parse |
| 309 | // today, and a FUTURE box's unknown keys (inside or alongside metrics) must |
| 310 | // not break today's parse either — additive wire shape, per AGENTS.md's |
| 311 | // "consumer = everything that PARSES the shape" rule. |
| 312 | func TestBoxBackCompatParse(t *testing.T) { |
| 313 | yesterday := []byte(`{"generated_at":"2026-08-03T10:00:00Z","label":"ada-stargazing", |
| 314 | "routes":{"static":1,"resident":2,"hibernating":3,"awake":null,"asleep":null}, |
| 315 | "load":"comfortable","capacity":"plenty", |
| 316 | "replication":{"healthy":true,"lag_seconds":5}, |
| 317 | "adoption_running":true,"expired_canaries":0}`) |
| 318 | var b status.Box |
| 319 | if err := json.Unmarshal(yesterday, &b); err != nil { |
| 320 | t.Fatalf("yesterday's shape failed to parse: %v", err) |
| 321 | } |
| 322 | if b.Metrics != nil { |
| 323 | t.Fatal("absent metrics must decode as nil, not zero-value") |
| 324 | } |
| 325 | tomorrow := []byte(`{"generated_at":"2026-08-05T10:00:00Z","label":"ada-stargazing", |
| 326 | "routes":{"static":0,"resident":0,"hibernating":0,"awake":null,"asleep":null}, |
| 327 | "load":null,"capacity":null,"replication":{"healthy":null,"lag_seconds":null}, |
| 328 | "adoption_running":null,"expired_canaries":null, |
| 329 | "metrics":{"mem_total_bytes":1,"some_future_field":true}}`) |
| 330 | if err := json.Unmarshal(tomorrow, &b); err != nil { |
| 331 | t.Fatalf("tomorrow's shape failed to parse: %v", err) |
| 332 | } |
| 333 | } |
| 334 | |
| 335 | // ---- Task 8: staged_version (route versions, private) ---------------------- |
| 336 | |
| 337 | // TestCollectBoxCarriesRouteVersions proves CollectBox attaches a host -> |
| 338 | // Version map onto the raw Metrics block for every non-platform route that |
| 339 | // carries a Version (registry.Route.Version, stamped by the adopt pass), and |
| 340 | // that WriteBox lands it in status/boxes/<label>.json — while BuildFleet's |
| 341 | // public fleet.json, which never copies Metrics by construction (see |
| 342 | // snapshot.go's package doc), carries no trace of the hostname. A platform- |
| 343 | // minted alias route and a route with no Version yet must not appear in the |
| 344 | // map at all. |
| 345 | func TestCollectBoxCarriesRouteVersions(t *testing.T) { |
| 346 | routes := []registry.Route{ |
| 347 | {Host: "h.test", Kind: "instance", Status: "ready", Version: "abc1234"}, |
| 348 | {Host: "unversioned.test", Kind: "instance", Status: "ready"}, |
| 349 | {Host: "platform.test", Kind: "instance", Status: "ready", Version: "zzz9999", Platform: true}, |
| 350 | } |
| 351 | b := status.CollectBox(routes, status.Probes{MachineID: "m"}, time.Now()) |
| 352 | if b.Metrics == nil { |
| 353 | t.Fatal("metrics block missing — RouteVersions should have attached it") |
| 354 | } |
| 355 | if len(b.Metrics.RouteVersions) != 1 || b.Metrics.RouteVersions["h.test"] != "abc1234" { |
| 356 | t.Fatalf("route_versions = %+v, want exactly {h.test: abc1234}", b.Metrics.RouteVersions) |
| 357 | } |
| 358 | |
| 359 | s := teststore(t) |
| 360 | ctx := context.Background() |
| 361 | if err := status.WriteBox(ctx, s, b); err != nil { |
| 362 | t.Fatal(err) |
| 363 | } |
| 364 | raw, _, err := s.Get(ctx, "status/boxes/"+b.Label+".json") |
| 365 | if err != nil { |
| 366 | t.Fatalf("Get written box: %v", err) |
| 367 | } |
| 368 | var round status.Box |
| 369 | if err := json.Unmarshal(raw, &round); err != nil { |
| 370 | t.Fatalf("unmarshal written box: %v", err) |
| 371 | } |
| 372 | if round.Metrics == nil || round.Metrics.RouteVersions["h.test"] != "abc1234" { |
| 373 | t.Errorf("written box route_versions = %+v, want {h.test: abc1234}: %s", round.Metrics, raw) |
| 374 | } |
| 375 | |
| 376 | f, err := status.BuildFleet(ctx, s, routes, time.Now(), 5*time.Minute) |
| 377 | if err != nil { |
| 378 | t.Fatal(err) |
| 379 | } |
| 380 | fj, err := json.Marshal(f) |
| 381 | if err != nil { |
| 382 | t.Fatal(err) |
| 383 | } |
| 384 | if strings.Contains(string(fj), "h.test") { |
| 385 | t.Errorf("fleet.json leaked a route_versions hostname: %s", fj) |
| 386 | } |
| 387 | } |
| 388 | |
| 389 | // TestCollectBoxRouteVersionsEmptyOmitsField proves a box with no versioned |
| 390 | // route contributes no route_versions key at all — omitted, not an empty |
| 391 | // object — matching the same "unknown/absent stays absent" convention every |
| 392 | // other Metrics field follows. |
| 393 | func TestCollectBoxRouteVersionsEmptyOmitsField(t *testing.T) { |
| 394 | routes := []registry.Route{{Host: "h.test", Kind: "instance", Status: "ready"}} |
| 395 | b := status.CollectBox(routes, status.Probes{MachineID: "m"}, time.Now()) |
| 396 | if b.Metrics != nil { |
| 397 | t.Fatalf("metrics block should stay nil with nothing else known, got %+v", b.Metrics) |
| 398 | } |
| 399 | } |
| 400 | |
| 401 | // ---- Task 6 (multi-upstream plan): per-upstream probe state ---------------- |
| 402 | |
| 403 | // TestCollectBoxCarriesUpstreams is route_versions' sibling for the probe |
| 404 | // table: CollectBox carries Probes.Upstreams onto the private Metrics block |
| 405 | // verbatim, WriteBox lands it in status/boxes/<label>.json, and BuildFleet's |
| 406 | // public fleet.json shows no trace of either the route host or the upstream |
| 407 | // ADDRESS — the most identifying pair this block has ever carried, and safe |
| 408 | // for exactly the same structural reason (BuildFleet never names Metrics). |
| 409 | func TestCollectBoxCarriesUpstreams(t *testing.T) { |
| 410 | routes := []registry.Route{{Host: "ha.test", Kind: "service", Status: "ready"}} |
| 411 | since := time.Now().Add(-time.Minute).UTC().Truncate(time.Second) |
| 412 | p := status.Probes{MachineID: "m", Upstreams: map[string][]status.UpstreamState{ |
| 413 | "ha.test": { |
| 414 | {Addr: "10.9.9.11:8080", Up: true, Since: since}, |
| 415 | {Addr: "10.9.9.12:8080", Up: false, Since: since, Fails: 4, LastError: "dial: connection refused"}, |
| 416 | }, |
| 417 | }} |
| 418 | |
| 419 | b := status.CollectBox(routes, p, time.Now()) |
| 420 | if b.Metrics == nil { |
| 421 | t.Fatal("metrics block missing — Upstreams should have attached it") |
| 422 | } |
| 423 | if len(b.Metrics.Upstreams) != 1 || len(b.Metrics.Upstreams["ha.test"]) != 2 { |
| 424 | t.Fatalf("upstreams = %+v, want one host with two upstreams", b.Metrics.Upstreams) |
| 425 | } |
| 426 | |
| 427 | s := teststore(t) |
| 428 | ctx := context.Background() |
| 429 | if err := status.WriteBox(ctx, s, b); err != nil { |
| 430 | t.Fatal(err) |
| 431 | } |
| 432 | raw, _, err := s.Get(ctx, "status/boxes/"+b.Label+".json") |
| 433 | if err != nil { |
| 434 | t.Fatalf("Get written box: %v", err) |
| 435 | } |
| 436 | var round status.Box |
| 437 | if err := json.Unmarshal(raw, &round); err != nil { |
| 438 | t.Fatalf("unmarshal written box: %v", err) |
| 439 | } |
| 440 | got := round.Metrics.Upstreams["ha.test"] |
| 441 | if !reflect.DeepEqual(got, p.Upstreams["ha.test"]) { |
| 442 | t.Errorf("written upstreams = %+v, want %+v: %s", got, p.Upstreams["ha.test"], raw) |
| 443 | } |
| 444 | |
| 445 | f, err := status.BuildFleet(ctx, s, routes, time.Now(), 5*time.Minute) |
| 446 | if err != nil { |
| 447 | t.Fatal(err) |
| 448 | } |
| 449 | fj, err := json.Marshal(f) |
| 450 | if err != nil { |
| 451 | t.Fatal(err) |
| 452 | } |
| 453 | for _, needle := range []string{"ha.test", "10.9.9.11:8080", "connection refused"} { |
| 454 | if strings.Contains(string(fj), needle) { |
| 455 | t.Errorf("fleet.json leaked %q from the private upstream table: %s", needle, fj) |
| 456 | } |
| 457 | } |
| 458 | } |
| 459 | |
| 460 | // TestCollectBoxUpstreamsEmptyOmitsField proves a box that probes nothing |
| 461 | // contributes no upstreams key at all — the same "absent, not empty" rule |
| 462 | // route_versions follows, which is what keeps every pre-existing snapshot |
| 463 | // consumer parsing a box written by this build unchanged. |
| 464 | func TestCollectBoxUpstreamsEmptyOmitsField(t *testing.T) { |
| 465 | routes := []registry.Route{{Host: "h.test", Kind: "instance", Status: "ready"}} |
| 466 | b := status.CollectBox(routes, status.Probes{MachineID: "m", Upstreams: map[string][]status.UpstreamState{}}, time.Now()) |
| 467 | if b.Metrics != nil { |
| 468 | t.Fatalf("an empty probe table must not attach a metrics block, got %+v", b.Metrics) |
| 469 | } |
| 470 | raw, err := json.Marshal(b) |
| 471 | if err != nil { |
| 472 | t.Fatal(err) |
| 473 | } |
| 474 | if strings.Contains(string(raw), "upstreams") { |
| 475 | t.Errorf("box JSON carries an upstreams key with nothing probed: %s", raw) |
| 476 | } |
| 477 | } |
| 478 | |
| 479 | // ---- Task 6: alias readings (route_aliases DNS/cert stamps, private) ----- |
| 480 | |
| 481 | // TestCollectBoxAliases proves CollectBox maps Probes.Aliases (registry rows |
| 482 | // carried in by the agent's own reg.ListAliases() call, see agent.go's |
| 483 | // statusTick) onto Box.Metrics.Aliases, keyed by the alias hostname — the |
| 484 | // same private home as RouteVersions (status/boxes/<label>.json only, never |
| 485 | // BuildFleet's public fleet.json) — and that an empty Probes.Aliases omits |
| 486 | // the "aliases" key entirely rather than publishing a present-but-empty map. |
| 487 | func TestCollectBoxAliases(t *testing.T) { |
| 488 | p := status.Probes{ |
| 489 | MachineID: "m", |
| 490 | Aliases: []registry.RouteAlias{ |
| 491 | {Alias: "extra.test", Host: "h.test", DNSStatus: "ok", CertExpiresAt: 1234567890}, |
| 492 | }, |
| 493 | } |
| 494 | b := status.CollectBox(nil, p, time.Now()) |
| 495 | if b.Metrics == nil { |
| 496 | t.Fatal("metrics block missing — Aliases should have attached it") |
| 497 | } |
| 498 | got, ok := b.Metrics.Aliases["extra.test"] |
| 499 | if !ok || got.Route != "h.test" || got.DNSStatus != "ok" || got.CertExpiresAt != 1234567890 { |
| 500 | t.Fatalf("aliases[extra.test] = %+v (ok=%v), want {Route:h.test DNSStatus:ok CertExpiresAt:1234567890}", got, ok) |
| 501 | } |
| 502 | |
| 503 | // Empty Probes.Aliases, but something else known so Metrics still |
| 504 | // attaches — the omission must be specific to the aliases key, not a |
| 505 | // side effect of the whole Metrics block being absent. |
| 506 | mt := int64(1) |
| 507 | b2 := status.CollectBox(nil, status.Probes{MachineID: "m", MemTotalBytes: &mt}, time.Now()) |
| 508 | j, err := json.Marshal(b2) |
| 509 | if err != nil { |
| 510 | t.Fatal(err) |
| 511 | } |
| 512 | if strings.Contains(string(j), "aliases") { |
| 513 | t.Fatalf("empty Probes.Aliases must omit the aliases key entirely, got: %s", j) |
| 514 | } |
| 515 | } |
| 516 | |
| 517 | // ---- Task 3: fleet merge + publisher -------------------------------------- |
| 518 | |
| 519 | // teststore is a fresh DirStore over a scratch directory — every fleet test |
| 520 | // gets an isolated bucket so writes in one test can never leak state into |
| 521 | // another. |
| 522 | func teststore(t *testing.T) release.Store { |
| 523 | t.Helper() |
| 524 | return release.NewDirStore(t.TempDir()) |
| 525 | } |
| 526 | |
| 527 | // boxFixture builds a minimal Box for tests that only care about label, |
| 528 | // capacity band, and generation time — the three things BuildFleet's |
| 529 | // worst-of/staleness logic actually looks at. |
| 530 | func boxFixture(label, capacity string, generatedAt time.Time) status.Box { |
| 531 | c := capacity |
| 532 | return status.Box{ |
| 533 | GeneratedAt: generatedAt, |
| 534 | Label: label, |
| 535 | Routes: status.RouteCounts{}, |
| 536 | Capacity: &c, |
| 537 | } |
| 538 | } |
| 539 | |
| 540 | func TestFleetAppsOptIn(t *testing.T) { |
| 541 | routes := []registry.Route{ |
| 542 | {Host: "kass.training", Kind: "instance", App: "kass", Status: "ready", |
| 543 | Channel: "stable", Version: "aedcae3", StatusNamed: true}, |
| 544 | {Host: "hidden.app", Kind: "instance", App: "hidden", Status: "ready"}, |
| 545 | {Host: "svc.pdf.internal", Kind: "service", App: "svc/pdf", Status: "ready"}, |
| 546 | } |
| 547 | s := teststore(t) // DirStore over t.TempDir(); write one Box via WriteBox first |
| 548 | if err := status.WriteBox(context.Background(), s, boxFixture("solo", "plenty", time.Now())); err != nil { |
| 549 | t.Fatal(err) |
| 550 | } |
| 551 | f, err := status.BuildFleet(context.Background(), s, routes, time.Now(), 5*time.Minute) |
| 552 | if err != nil { |
| 553 | t.Fatal(err) |
| 554 | } |
| 555 | if len(f.Apps) != 1 || f.Apps[0].Name != "kass" || f.Apps[0].Version != "aedcae3" { |
| 556 | t.Errorf("apps = %+v, want exactly the opted-in kass", f.Apps) |
| 557 | } |
| 558 | if f.AppsUnlisted != 1 { // hidden; svc/* routes are services, not apps |
| 559 | t.Errorf("apps_unlisted = %d, want 1", f.AppsUnlisted) |
| 560 | } |
| 561 | if len(f.Services) != 1 || f.Services[0].Name != "svc/pdf" || !f.Services[0].Up { |
| 562 | t.Errorf("services = %+v", f.Services) |
| 563 | } |
| 564 | j, _ := json.Marshal(f) |
| 565 | if strings.Contains(string(j), "hidden") { |
| 566 | t.Errorf("un-opted app leaked: %s", j) |
| 567 | } |
| 568 | } |
| 569 | |
| 570 | // App names are unique per ACCOUNT, never globally, so two tenants can both |
| 571 | // run a "blog". Grouping on the name alone would merge them — publishing one |
| 572 | // tenant's outage under the other's name, and letting either tenant's opt-in |
| 573 | // decide whether the other appears at all. |
| 574 | func TestFleetAppsAreGroupedPerAccount(t *testing.T) { |
| 575 | routes := []registry.Route{ |
| 576 | {Host: "blog.one", Kind: "instance", App: "blog", Account: "bab", Status: "ready", |
| 577 | Channel: "stable", Version: "v1", StatusNamed: true}, |
| 578 | {Host: "blog.two", Kind: "instance", App: "blog", Account: "kqmv", Status: "failed", |
| 579 | Channel: "stable", Version: "v9", StatusNamed: true}, |
| 580 | } |
| 581 | s := teststore(t) |
| 582 | if err := status.WriteBox(context.Background(), s, boxFixture("solo", "plenty", time.Now())); err != nil { |
| 583 | t.Fatal(err) |
| 584 | } |
| 585 | f, err := status.BuildFleet(context.Background(), s, routes, time.Now(), 5*time.Minute) |
| 586 | if err != nil { |
| 587 | t.Fatal(err) |
| 588 | } |
| 589 | if len(f.Apps) != 2 { |
| 590 | t.Fatalf("apps = %+v, want two separate blogs (one per account)", f.Apps) |
| 591 | } |
| 592 | byVersion := map[string]bool{} |
| 593 | for _, a := range f.Apps { |
| 594 | if a.Name != "blog" { |
| 595 | t.Errorf("app name = %q, want the app name (never the sqid)", a.Name) |
| 596 | } |
| 597 | byVersion[a.Version] = a.Healthy |
| 598 | } |
| 599 | if !byVersion["v1"] { |
| 600 | t.Error("the healthy account's blog was dragged down by the other's") |
| 601 | } |
| 602 | if byVersion["v9"] { |
| 603 | t.Error("the failing account's blog was reported healthy") |
| 604 | } |
| 605 | } |
| 606 | |
| 607 | // A tenant that did NOT opt in must stay dark even when another account's |
| 608 | // same-named app did — the opt-in is per app, and merging would publish an |
| 609 | // app that never asked to be named. |
| 610 | func TestFleetOptInIsPerAccount(t *testing.T) { |
| 611 | routes := []registry.Route{ |
| 612 | {Host: "blog.one", Kind: "instance", App: "blog", Account: "bab", Status: "ready", |
| 613 | Channel: "stable", Version: "v1", StatusNamed: true}, |
| 614 | {Host: "blog.two", Kind: "instance", App: "blog", Account: "kqmv", Status: "ready", |
| 615 | Channel: "stable", Version: "secret", StatusNamed: false}, |
| 616 | } |
| 617 | s := teststore(t) |
| 618 | if err := status.WriteBox(context.Background(), s, boxFixture("solo", "plenty", time.Now())); err != nil { |
| 619 | t.Fatal(err) |
| 620 | } |
| 621 | f, err := status.BuildFleet(context.Background(), s, routes, time.Now(), 5*time.Minute) |
| 622 | if err != nil { |
| 623 | t.Fatal(err) |
| 624 | } |
| 625 | if len(f.Apps) != 1 || f.Apps[0].Version != "v1" { |
| 626 | t.Fatalf("apps = %+v, want only the opted-in account's blog", f.Apps) |
| 627 | } |
| 628 | if f.AppsUnlisted != 1 { |
| 629 | t.Errorf("apps_unlisted = %d, want 1", f.AppsUnlisted) |
| 630 | } |
| 631 | j, _ := json.Marshal(f) |
| 632 | if strings.Contains(string(j), "secret") { |
| 633 | t.Errorf("un-opted app leaked: %s", j) |
| 634 | } |
| 635 | } |
| 636 | |
| 637 | func TestFleetWorstOfAndStale(t *testing.T) { |
| 638 | s := teststore(t) |
| 639 | now := time.Now() |
| 640 | old, fresh := boxFixture("old", "at-capacity", now.Add(-16*time.Minute)), |
| 641 | boxFixture("fresh", "plenty", now) // helper: Box with that capacity word |
| 642 | for _, b := range []status.Box{old, fresh} { |
| 643 | if err := status.WriteBox(context.Background(), s, b); err != nil { |
| 644 | t.Fatal(err) |
| 645 | } |
| 646 | } |
| 647 | f, _ := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| 648 | if f.Capacity.Level == nil || *f.Capacity.Level != "at-capacity" { |
| 649 | t.Errorf("capacity must be the worst box's band, got %v", f.Capacity.Level) |
| 650 | } |
| 651 | var st, fr *status.FleetBox |
| 652 | for i := range f.Boxes { |
| 653 | if f.Boxes[i].Label == "old" { |
| 654 | st = &f.Boxes[i] |
| 655 | } else { |
| 656 | fr = &f.Boxes[i] |
| 657 | } |
| 658 | } |
| 659 | if !st.Stale || fr.Stale { |
| 660 | t.Error("boxes older than 3 periods must mark stale — loudly, never dropped") |
| 661 | } |
| 662 | } |
| 663 | |
| 664 | // TestFleetInstanceSumsNullPropagate proves the "unknowns are null, never 0" |
| 665 | // rule holds through aggregation too: one box with an unreadable socket |
| 666 | // table (Awake/Asleep null) must null the FLEET-WIDE awake/asleep sum, not |
| 667 | // silently drop that box's contribution and report a partial — and |
| 668 | // misleadingly confident — total. |
| 669 | func TestFleetInstanceSumsNullPropagate(t *testing.T) { |
| 670 | s := teststore(t) |
| 671 | now := time.Now() |
| 672 | known := status.CollectBox( |
| 673 | []registry.Route{{Host: "a", Kind: "instance", Hibernate: true, Socket: "/a.sock"}}, |
| 674 | status.Probes{MachineID: "known", Listening: func(string) *bool { v := true; return &v }}, |
| 675 | now, |
| 676 | ) |
| 677 | unknown := status.CollectBox( |
| 678 | []registry.Route{{Host: "b", Kind: "instance", Hibernate: true, Socket: "/b.sock"}}, |
| 679 | status.Probes{MachineID: "unknown"}, // nil Listening -> Awake/Asleep null |
| 680 | now, |
| 681 | ) |
| 682 | for _, b := range []status.Box{known, unknown} { |
| 683 | if err := status.WriteBox(context.Background(), s, b); err != nil { |
| 684 | t.Fatal(err) |
| 685 | } |
| 686 | } |
| 687 | f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| 688 | if err != nil { |
| 689 | t.Fatal(err) |
| 690 | } |
| 691 | if f.Instances["awake"] != nil { |
| 692 | t.Errorf("fleet awake = %v, want nil (one box's socket table was unreadable)", *f.Instances["awake"]) |
| 693 | } |
| 694 | if f.Instances["static"] == nil || *f.Instances["static"] != 0 { |
| 695 | t.Errorf("fleet static = %v, want 0 (a known, non-null component)", f.Instances["static"]) |
| 696 | } |
| 697 | if f.Instances["hibernating"] == nil || *f.Instances["hibernating"] != 2 { |
| 698 | t.Errorf("fleet hibernating = %v, want 2", f.Instances["hibernating"]) |
| 699 | } |
| 700 | } |
| 701 | |
| 702 | // TestFleetEmptyIsValid locks down the zero-snapshot state: BuildFleet must |
| 703 | // not error when status/boxes/ is empty, and must report zero counts (not |
| 704 | // null — there's nothing unknown about an empty fleet) with a null capacity |
| 705 | // band (there's no box to have voted on one). This is the console's |
| 706 | // "no snapshots yet" rendering. |
| 707 | func TestFleetEmptyIsValid(t *testing.T) { |
| 708 | s := teststore(t) |
| 709 | f, err := status.BuildFleet(context.Background(), s, nil, time.Now(), 5*time.Minute) |
| 710 | if err != nil { |
| 711 | t.Fatal(err) |
| 712 | } |
| 713 | if len(f.Boxes) != 0 { |
| 714 | t.Errorf("boxes = %+v, want none", f.Boxes) |
| 715 | } |
| 716 | if f.Capacity.Level != nil { |
| 717 | t.Errorf("capacity.level = %v, want nil with no boxes to vote", *f.Capacity.Level) |
| 718 | } |
| 719 | if f.Instances["total"] == nil || *f.Instances["total"] != 0 { |
| 720 | t.Errorf("instances.total = %v, want 0", f.Instances["total"]) |
| 721 | } |
| 722 | if len(f.Apps) != 0 || f.AppsUnlisted != 0 || len(f.Services) != 0 { |
| 723 | t.Errorf("apps/services must be empty with no routes, got apps=%+v unlisted=%d services=%+v", f.Apps, f.AppsUnlisted, f.Services) |
| 724 | } |
| 725 | } |
| 726 | |
| 727 | // TestFleetSkipsCorruptSnapshot proves a corrupt status/boxes/*.json is a |
| 728 | // logged skip, never a fatal error — one box's bad write must not take the |
| 729 | // whole fleet page down. |
| 730 | func TestFleetSkipsCorruptSnapshot(t *testing.T) { |
| 731 | s := teststore(t) |
| 732 | now := time.Now() |
| 733 | if err := status.WriteBox(context.Background(), s, boxFixture("good", "plenty", now)); err != nil { |
| 734 | t.Fatal(err) |
| 735 | } |
| 736 | if _, err := s.Put(context.Background(), "status/boxes/broken.json", []byte("{not json"), ""); err != nil { |
| 737 | t.Fatal(err) |
| 738 | } |
| 739 | f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| 740 | if err != nil { |
| 741 | t.Fatalf("BuildFleet must not fail on a corrupt snapshot: %v", err) |
| 742 | } |
| 743 | if len(f.Boxes) != 1 || f.Boxes[0].Label != "good" { |
| 744 | t.Errorf("boxes = %+v, want only the good one (corrupt skipped)", f.Boxes) |
| 745 | } |
| 746 | } |
| 747 | |
| 748 | // TestFleetLoadBoxesCapsKeyCount is I3 (final customer-fleets review): |
| 749 | // status/boxes/ is customer-writable in the fleet case (via |
| 750 | // CARLOS_STATUS_PREFIX), so loadBoxes must bound how many keys it will ever |
| 751 | // process from one listing rather than trusting an untrusted prefix's size. |
| 752 | // Seed one more box than the cap and confirm only the cap's worth land in |
| 753 | // the merged Fleet. |
| 754 | func TestFleetLoadBoxesCapsKeyCount(t *testing.T) { |
| 755 | s := teststore(t) |
| 756 | now := time.Now() |
| 757 | const keyCap = 200 |
| 758 | for i := 0; i < keyCap+1; i++ { |
| 759 | label := fmt.Sprintf("box-%04d", i) |
| 760 | if err := status.WriteBox(context.Background(), s, boxFixture(label, "plenty", now)); err != nil { |
| 761 | t.Fatal(err) |
| 762 | } |
| 763 | } |
| 764 | f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| 765 | if err != nil { |
| 766 | t.Fatalf("BuildFleet must not fail on an oversized key listing: %v", err) |
| 767 | } |
| 768 | if len(f.Boxes) != keyCap { |
| 769 | t.Fatalf("boxes = %d, want exactly the %d-key cap (seeded %d)", len(f.Boxes), keyCap, keyCap+1) |
| 770 | } |
| 771 | } |
| 772 | |
| 773 | // TestFleetLoadBoxesSkipsOversizedBody is I3's other half: a single huge |
| 774 | // object under status/boxes/ (a hostile or misbehaving box parking a large |
| 775 | // body on its own customer-writable prefix) must be skipped, logged, and |
| 776 | // must not prevent a sibling box's snapshot from loading. |
| 777 | func TestFleetLoadBoxesSkipsOversizedBody(t *testing.T) { |
| 778 | s := teststore(t) |
| 779 | now := time.Now() |
| 780 | if err := status.WriteBox(context.Background(), s, boxFixture("good", "plenty", now)); err != nil { |
| 781 | t.Fatal(err) |
| 782 | } |
| 783 | huge := make([]byte, (1<<20)+1) // one byte over the 1MB cap |
| 784 | for i := range huge { |
| 785 | huge[i] = ' ' |
| 786 | } |
| 787 | if _, err := s.Put(context.Background(), "status/boxes/huge.json", huge, ""); err != nil { |
| 788 | t.Fatal(err) |
| 789 | } |
| 790 | f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| 791 | if err != nil { |
| 792 | t.Fatalf("BuildFleet must not fail on an oversized body: %v", err) |
| 793 | } |
| 794 | if len(f.Boxes) != 1 || f.Boxes[0].Label != "good" { |
| 795 | t.Errorf("boxes = %+v, want only the good one (oversized body skipped)", f.Boxes) |
| 796 | } |
| 797 | } |
| 798 | |
| 799 | // TestPublishFleetWritesAndOverwrites mirrors WriteBox's own overwrite test: |
| 800 | // PublishFleet lands status/fleet.json on every publish pass (every 5 |
| 801 | // minutes, per the plan), so a second call must overwrite cleanly rather |
| 802 | // than failing with ErrCASConflict against release.Store's create-only Put. |
| 803 | func TestPublishFleetWritesAndOverwrites(t *testing.T) { |
| 804 | s := teststore(t) |
| 805 | ctx := context.Background() |
| 806 | |
| 807 | first := &status.Fleet{GeneratedAt: time.Unix(1000, 0)} |
| 808 | if err := status.PublishFleet(ctx, s, first); err != nil { |
| 809 | t.Fatalf("first PublishFleet: %v", err) |
| 810 | } |
| 811 | second := &status.Fleet{GeneratedAt: time.Unix(2000, 0)} |
| 812 | if err := status.PublishFleet(ctx, s, second); err != nil { |
| 813 | t.Fatalf("second PublishFleet (must overwrite, not conflict): %v", err) |
| 814 | } |
| 815 | |
| 816 | got, _, err := s.Get(ctx, "status/fleet.json") |
| 817 | if err != nil { |
| 818 | t.Fatalf("Get status/fleet.json: %v", err) |
| 819 | } |
| 820 | var round status.Fleet |
| 821 | if err := json.Unmarshal(got, &round); err != nil { |
| 822 | t.Fatalf("unmarshal published fleet: %v", err) |
| 823 | } |
| 824 | if !round.GeneratedAt.Equal(second.GeneratedAt) { |
| 825 | t.Errorf("stored fleet has GeneratedAt %v, want second publish's %v", round.GeneratedAt, second.GeneratedAt) |
| 826 | } |
| 827 | } |
| 828 | |
| 829 | // ---- Task 4: embedded viewer page ----------------------------------------- |
| 830 | |
| 831 | // TestPublishWritesPageAssets proves PublishFleet ships the whole viewer — |
| 832 | // fleet.json alongside the embedded index.html and status.css — and that the |
| 833 | // page is same-origin only: no absolute http(s):// URL and no <script src> |
| 834 | // anywhere in the shipped markup. (Simplified from the plan's step-1 text, |
| 835 | // per the operator's ambiguity resolution: the footer's incident-fallback |
| 836 | // URL is injected as text by inline JS from `location` at view time, never |
| 837 | // baked into the HTML as a hardcoded absolute href, so a flat "no |
| 838 | // http(s):// substring, no <script src" check is the honest same-origin |
| 839 | // assertion here — no carve-out needed for a rel="me" link that doesn't |
| 840 | // exist.) |
| 841 | func TestPublishWritesPageAssets(t *testing.T) { |
| 842 | s := teststore(t) |
| 843 | f, err := status.BuildFleet(context.Background(), s, nil, time.Now(), 5*time.Minute) |
| 844 | if err != nil { |
| 845 | t.Fatal(err) |
| 846 | } |
| 847 | if err := status.PublishFleet(context.Background(), s, f); err != nil { |
| 848 | t.Fatal(err) |
| 849 | } |
| 850 | for _, key := range []string{"status/fleet.json", "status/index.html", "status/status.css"} { |
| 851 | if _, _, err := s.Get(context.Background(), key); err != nil { |
| 852 | t.Errorf("%s not published: %v", key, err) |
| 853 | } |
| 854 | } |
| 855 | html, _, err := s.Get(context.Background(), "status/index.html") |
| 856 | if err != nil { |
| 857 | t.Fatalf("Get status/index.html: %v", err) |
| 858 | } |
| 859 | // The last three entries catch protocol-relative references |
| 860 | // (src="//cdn...", href="//cdn...", url(//cdn...)) — another way to |
| 861 | // reach off-origin without ever spelling "http". A bare "//" substring |
| 862 | // isn't checked: this file's inline script is full of legitimate "// |
| 863 | // comment" lines, so the assertion targets the actual reference |
| 864 | // positions (an attribute/url() opening onto "//") instead. |
| 865 | for _, banned := range []string{"http://", "https://", "<script src", `src="//`, `href="//`, `url(//`} { |
| 866 | if strings.Contains(string(html), banned) { |
| 867 | t.Errorf("page must be same-origin only; found %q in index.html", banned) |
| 868 | } |
| 869 | } |
| 870 | } |
| 871 | |
| 872 | // TestWriteBoxReturnsErrorOnDoubleCASConflict proves WriteBox does NOT retry |
| 873 | // a second time: if the retried Put also loses the race, that error must |
| 874 | // reach the caller rather than being swallowed or retried forever. |
| 875 | func TestWriteBoxReturnsErrorOnDoubleCASConflict(t *testing.T) { |
| 876 | dir := t.TempDir() |
| 877 | real := release.NewDirStore(dir) |
| 878 | flaky := &flakyPutStore{Store: real, failCount: 2} |
| 879 | b := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Now()) |
| 880 | |
| 881 | err := status.WriteBox(context.Background(), flaky, b) |
| 882 | if err == nil { |
| 883 | t.Fatal("WriteBox: want an error after two consecutive CAS conflicts, got nil") |
| 884 | } |
| 885 | if !errors.Is(err, release.ErrCASConflict) { |
| 886 | t.Errorf("WriteBox error = %v, want it to wrap release.ErrCASConflict", err) |
| 887 | } |
| 888 | if flaky.putCalls != 2 { |
| 889 | t.Errorf("Put called %d times, want exactly 2 (no third attempt)", flaky.putCalls) |
| 890 | } |
| 891 | } |
| 892 | |
| 893 | // ---- Task 6: dry-run self-check -------------------------------------------- |
| 894 | |
| 895 | // TestDryRunSelfChecks is the plan's Step-1 test verbatim: DryRun must run |
| 896 | // the real CollectBox -> WriteBox -> BuildFleet -> PublishFleet path over a |
| 897 | // fixture registry stuffed with needles (hostnames, a machine id, mid-band |
| 898 | // fractions), then read every published object back and prove none of those |
| 899 | // needles survived — the self-hoster's "what would I publish?" check, run |
| 900 | // before ever pointing this binary at a real bucket. |
| 901 | func TestDryRunSelfChecks(t *testing.T) { |
| 902 | var buf bytes.Buffer |
| 903 | if err := status.DryRun(t.TempDir(), &buf); err != nil { |
| 904 | t.Fatalf("dry run: %v\n%s", err, buf.String()) |
| 905 | } |
| 906 | if !strings.Contains(buf.String(), "no leaks") { |
| 907 | t.Errorf("dry run must attest the self-check ran: %s", buf.String()) |
| 908 | } |
| 909 | } |
| 910 | |
| 911 | // ---- Pipelines Task 5: adoption health ------------------------------------- |
| 912 | |
| 913 | // TestAdoptionUnknownIsANull pins the honest-unknown half of the pipelines |
| 914 | // spec's §5 ("adoption must actually run"): a box whose probe never managed to |
| 915 | // ask systemd about the adoption timer publishes null, exactly like every |
| 916 | // other unmeasured quantity in this package. The failure this guards against |
| 917 | // is not a wrong reading, it is a REASSURING one — a default true would make |
| 918 | // every box that can't answer look like a box that is adopting fine, which is |
| 919 | // precisely the silence that let carlos-adopt.timer sit enabled-but-never- |
| 920 | // started on the flagship for weeks with nothing on any page to show for it. |
| 921 | func TestAdoptionUnknownIsANull(t *testing.T) { |
| 922 | b := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Now()) // nil AdoptRunning |
| 923 | if b.AdoptionRunning != nil { |
| 924 | t.Errorf("adoption_running = %v, want null — an unasked question is not a healthy answer", *b.AdoptionRunning) |
| 925 | } |
| 926 | for _, want := range []bool{true, false} { |
| 927 | got := status.CollectBox(nil, status.Probes{MachineID: "m", AdoptRunning: &want}, time.Now()).AdoptionRunning |
| 928 | if got == nil || *got != want { |
| 929 | t.Errorf("AdoptRunning=%v: adoption_running = %v, want %v", want, got, want) |
| 930 | } |
| 931 | } |
| 932 | } |
| 933 | |
| 934 | // TestConfigPathRunningTravelsPerBox pins the issue #44 probe's whole chain: |
| 935 | // whether this box's config-sync trigger unit (carlos-adopt.path) is ACTIVE |
| 936 | // travels from Probes onto the box snapshot and through BuildFleet onto the |
| 937 | // box's public row — the same journey adoption_running makes, and for the |
| 938 | // same reason. An inactive path unit is a box that silently fell back to the |
| 939 | // hourly config floor, which is exactly the class of quiet degradation this |
| 940 | // package exists to make visible. Unknown stays null: an unasked question is |
| 941 | // not a reassuring answer here either. |
| 942 | func TestConfigPathRunningTravelsPerBox(t *testing.T) { |
| 943 | if b := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Now()); b.ConfigPathRunning != nil { |
| 944 | t.Errorf("config_path_running = %v, want null when the probe never answered", *b.ConfigPathRunning) |
| 945 | } |
| 946 | for _, want := range []bool{true, false} { |
| 947 | got := status.CollectBox(nil, status.Probes{MachineID: "m", ConfigPathRunning: &want}, time.Now()).ConfigPathRunning |
| 948 | if got == nil || *got != want { |
| 949 | t.Fatalf("ConfigPathRunning=%v: box field = %v, want %v", want, got, want) |
| 950 | } |
| 951 | } |
| 952 | |
| 953 | s := teststore(t) |
| 954 | now := time.Now() |
| 955 | no := false |
| 956 | b := status.CollectBox(nil, status.Probes{MachineID: "m", ConfigPathRunning: &no}, now) |
| 957 | if err := status.WriteBox(context.Background(), s, b); err != nil { |
| 958 | t.Fatal(err) |
| 959 | } |
| 960 | f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| 961 | if err != nil { |
| 962 | t.Fatal(err) |
| 963 | } |
| 964 | if len(f.Boxes) != 1 { |
| 965 | t.Fatalf("fleet boxes = %d, want 1", len(f.Boxes)) |
| 966 | } |
| 967 | if got := f.Boxes[0].ConfigPathRunning; got == nil || *got { |
| 968 | t.Fatalf("fleet box config_path_running = %v, want false — knowing WHICH box lost its trigger is the whole value", got) |
| 969 | } |
| 970 | } |
| 971 | |
| 972 | // TestConfigPathRunningIsWireAdditive pins the other half: a snapshot written |
| 973 | // before this field existed (no "config_path_running" key at all) must still |
| 974 | // parse, and must read as unknown rather than as a healthy false. Live |
| 975 | // buckets hold exactly those objects today. |
| 976 | func TestConfigPathRunningIsWireAdditive(t *testing.T) { |
| 977 | var b status.Box |
| 978 | old := `{"generated_at":"2026-08-04T00:00:00Z","label":"box-old","routes":{"static":0,"resident":0,"hibernating":0,"awake":null,"asleep":null},"load":null,"capacity":null,"replication":{"healthy":null,"lag_seconds":null},"adoption_running":true,"expired_canaries":null}` |
| 979 | if err := json.Unmarshal([]byte(old), &b); err != nil { |
| 980 | t.Fatalf("a pre-#44 snapshot must still parse: %v", err) |
| 981 | } |
| 982 | if b.ConfigPathRunning != nil { |
| 983 | t.Fatalf("config_path_running = %v, want null for a snapshot written before the field existed", *b.ConfigPathRunning) |
| 984 | } |
| 985 | } |
| 986 | |
| 987 | // TestRestartPathRunningTravelsPerBox is TestConfigPathRunningTravelsPerBox's |
| 988 | // issue #113 twin, and it exists for a sharper reason than symmetry: with |
| 989 | // carlos-restart.path unarmed, a member runs `carlos restart` at a wedged |
| 990 | // instance, is told "requested", and waits for a cycle that will not come until |
| 991 | // the hourly pass. Nothing inside the agent can see that — it writes its marker |
| 992 | // either way — so this reading is the only thing that can. Unknown stays null. |
| 993 | func TestRestartPathRunningTravelsPerBox(t *testing.T) { |
| 994 | if b := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Now()); b.RestartPathRunning != nil { |
| 995 | t.Errorf("restart_path_running = %v, want null when the probe never answered", *b.RestartPathRunning) |
| 996 | } |
| 997 | for _, want := range []bool{true, false} { |
| 998 | got := status.CollectBox(nil, status.Probes{MachineID: "m", RestartPathRunning: &want}, time.Now()).RestartPathRunning |
| 999 | if got == nil || *got != want { |
| 1000 | t.Fatalf("RestartPathRunning=%v: box field = %v, want %v", want, got, want) |
| 1001 | } |
| 1002 | } |
| 1003 | |
| 1004 | s := teststore(t) |
| 1005 | now := time.Now() |
| 1006 | no := false |
| 1007 | b := status.CollectBox(nil, status.Probes{MachineID: "m", RestartPathRunning: &no}, now) |
| 1008 | if err := status.WriteBox(context.Background(), s, b); err != nil { |
| 1009 | t.Fatal(err) |
| 1010 | } |
| 1011 | f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| 1012 | if err != nil { |
| 1013 | t.Fatal(err) |
| 1014 | } |
| 1015 | if len(f.Boxes) != 1 { |
| 1016 | t.Fatalf("fleet boxes = %d, want 1", len(f.Boxes)) |
| 1017 | } |
| 1018 | if got := f.Boxes[0].RestartPathRunning; got == nil || *got { |
| 1019 | t.Fatalf("fleet box restart_path_running = %v, want false — knowing WHICH box cannot cycle a wedged instance is the whole value", got) |
| 1020 | } |
| 1021 | |
| 1022 | // Wire-additive, like every field before it: a snapshot written before |
| 1023 | // issue #113 has no such key and must read as unknown, not a healthy false. |
| 1024 | var old status.Box |
| 1025 | raw := `{"generated_at":"2026-08-04T00:00:00Z","label":"box-old","routes":{"static":0,"resident":0,"hibernating":0,"awake":null,"asleep":null},"load":null,"capacity":null,"replication":{"healthy":null,"lag_seconds":null},"adoption_running":true,"config_path_running":true,"expired_canaries":null}` |
| 1026 | if err := json.Unmarshal([]byte(raw), &old); err != nil { |
| 1027 | t.Fatalf("a pre-#113 snapshot must still parse: %v", err) |
| 1028 | } |
| 1029 | if old.RestartPathRunning != nil { |
| 1030 | t.Fatalf("restart_path_running = %v, want null for a snapshot written before the field existed", *old.RestartPathRunning) |
| 1031 | } |
| 1032 | } |
| 1033 | |
| 1034 | // TestFleetAdoptionWorstOf covers the aggregate: a box that is KNOWN not to be |
| 1035 | // adopting is a defect the fleet number must never hide, so one false outranks |
| 1036 | // any number of trues; an unknown box can't be summed into a confident true, |
| 1037 | // so it nulls the aggregate instead; and a stale box doesn't vote at all |
| 1038 | // (staleness means "we no longer know", the same rule worstReplication uses). |
| 1039 | func TestFleetAdoptionWorstOf(t *testing.T) { |
| 1040 | yes, no := true, false |
| 1041 | cases := []struct { |
| 1042 | name string |
| 1043 | boxes map[string]*bool // machine id -> AdoptRunning |
| 1044 | stale map[string]bool // machine id -> written 16 minutes ago |
| 1045 | want *bool |
| 1046 | }{ |
| 1047 | {"all running", map[string]*bool{"a": &yes, "b": &yes}, nil, &yes}, |
| 1048 | {"one dead timer wins", map[string]*bool{"a": &yes, "b": &no}, nil, &no}, |
| 1049 | {"one unknown blocks a cheerful true", map[string]*bool{"a": &yes, "b": nil}, nil, nil}, |
| 1050 | {"a known defect beats an unknown", map[string]*bool{"a": nil, "b": &no}, nil, &no}, |
| 1051 | {"stale boxes do not vote", map[string]*bool{"a": &yes, "b": &no}, map[string]bool{"b": true}, &yes}, |
| 1052 | {"no boxes at all", nil, nil, nil}, |
| 1053 | } |
| 1054 | for _, c := range cases { |
| 1055 | t.Run(c.name, func(t *testing.T) { |
| 1056 | s := teststore(t) |
| 1057 | now := time.Now() |
| 1058 | for id, adopt := range c.boxes { |
| 1059 | at := now |
| 1060 | if c.stale[id] { |
| 1061 | at = now.Add(-16 * time.Minute) // > 3 x the 5-minute period |
| 1062 | } |
| 1063 | b := status.CollectBox(nil, status.Probes{MachineID: id, AdoptRunning: adopt}, at) |
| 1064 | if err := status.WriteBox(context.Background(), s, b); err != nil { |
| 1065 | t.Fatal(err) |
| 1066 | } |
| 1067 | } |
| 1068 | f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| 1069 | if err != nil { |
| 1070 | t.Fatal(err) |
| 1071 | } |
| 1072 | switch { |
| 1073 | case c.want == nil && f.AdoptionRunning != nil: |
| 1074 | t.Errorf("fleet adoption_running = %v, want null", *f.AdoptionRunning) |
| 1075 | case c.want != nil && f.AdoptionRunning == nil: |
| 1076 | t.Errorf("fleet adoption_running = null, want %v", *c.want) |
| 1077 | case c.want != nil && *f.AdoptionRunning != *c.want: |
| 1078 | t.Errorf("fleet adoption_running = %v, want %v", *f.AdoptionRunning, *c.want) |
| 1079 | } |
| 1080 | }) |
| 1081 | } |
| 1082 | } |
| 1083 | |
| 1084 | // TestFleetBoxCarriesAdoption keeps the per-box row honest too: the fleet-wide |
| 1085 | // number says SOMETHING is not adopting, the boxes table says which label — |
| 1086 | // without it, an operator reading "not running" on a two-box fleet has to go |
| 1087 | // SSH into both to find out where the dead timer is. |
| 1088 | // ---- Pipelines Task: canary TTL (spec §5 "Canaries expire") --------------- |
| 1089 | |
| 1090 | // TestExpiredCanariesUnknownIsANull mirrors TestAdoptionUnknownIsANull: a box |
| 1091 | // whose probe never resolved its canary channel pointers (e.g. no deployment |
| 1092 | // store configured) publishes null, exactly like every other unmeasured |
| 1093 | // quantity here. A default 0 would read as "no expired canaries on this |
| 1094 | // box" when the honest answer is "this box couldn't check" — precisely the |
| 1095 | // reassuring-wrong-answer failure the rest of this package refuses to make. |
| 1096 | func TestExpiredCanariesUnknownIsANull(t *testing.T) { |
| 1097 | b := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Now()) // nil ExpiredCanaries |
| 1098 | if b.ExpiredCanaries != nil { |
| 1099 | t.Errorf("expired_canaries = %v, want null — an unasked question is not a healthy answer", *b.ExpiredCanaries) |
| 1100 | } |
| 1101 | for _, want := range []int{0, 3} { |
| 1102 | got := status.CollectBox(nil, status.Probes{MachineID: "m", ExpiredCanaries: &want}, time.Now()).ExpiredCanaries |
| 1103 | if got == nil || *got != want { |
| 1104 | t.Errorf("ExpiredCanaries=%d: expired_canaries = %v, want %d", want, got, want) |
| 1105 | } |
| 1106 | } |
| 1107 | } |
| 1108 | |
| 1109 | // TestFleetExpiredCanariesSums proves the fleet-wide count is the sum across |
| 1110 | // boxes when every box's own count is known — the aggregate an operator |
| 1111 | // actually wants ("how many expired canaries are running anywhere"), not a |
| 1112 | // per-box worst-of. |
| 1113 | func TestFleetExpiredCanariesSums(t *testing.T) { |
| 1114 | s := teststore(t) |
| 1115 | now := time.Now() |
| 1116 | two, one := 2, 1 |
| 1117 | for _, b := range []status.Box{ |
| 1118 | status.CollectBox(nil, status.Probes{MachineID: "a", ExpiredCanaries: &two}, now), |
| 1119 | status.CollectBox(nil, status.Probes{MachineID: "b", ExpiredCanaries: &one}, now), |
| 1120 | } { |
| 1121 | if err := status.WriteBox(context.Background(), s, b); err != nil { |
| 1122 | t.Fatal(err) |
| 1123 | } |
| 1124 | } |
| 1125 | f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| 1126 | if err != nil { |
| 1127 | t.Fatal(err) |
| 1128 | } |
| 1129 | if f.ExpiredCanaries == nil || *f.ExpiredCanaries != 3 { |
| 1130 | t.Errorf("fleet expired_canaries = %v, want 3", f.ExpiredCanaries) |
| 1131 | } |
| 1132 | } |
| 1133 | |
| 1134 | // TestFleetExpiredCanariesNullPropagates: one box that couldn't check nulls |
| 1135 | // the fleet-wide sum, the same null-propagation rule sumInstances applies to |
| 1136 | // Awake/Asleep — a partial sum would read as more certain than it is. |
| 1137 | func TestFleetExpiredCanariesNullPropagates(t *testing.T) { |
| 1138 | s := teststore(t) |
| 1139 | now := time.Now() |
| 1140 | two := 2 |
| 1141 | known := status.CollectBox(nil, status.Probes{MachineID: "known", ExpiredCanaries: &two}, now) |
| 1142 | unknown := status.CollectBox(nil, status.Probes{MachineID: "unknown"}, now) // nil probe |
| 1143 | for _, b := range []status.Box{known, unknown} { |
| 1144 | if err := status.WriteBox(context.Background(), s, b); err != nil { |
| 1145 | t.Fatal(err) |
| 1146 | } |
| 1147 | } |
| 1148 | f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| 1149 | if err != nil { |
| 1150 | t.Fatal(err) |
| 1151 | } |
| 1152 | if f.ExpiredCanaries != nil { |
| 1153 | t.Errorf("fleet expired_canaries = %v, want nil (one box could not check)", *f.ExpiredCanaries) |
| 1154 | } |
| 1155 | } |
| 1156 | |
| 1157 | // TestFleetEmptyExpiredCanariesIsZero: no boxes at all is not "unknown", the |
| 1158 | // same rule TestFleetEmptyIsValid pins for instances — there is nothing |
| 1159 | // unknown about an empty fleet. |
| 1160 | func TestFleetEmptyExpiredCanariesIsZero(t *testing.T) { |
| 1161 | s := teststore(t) |
| 1162 | f, err := status.BuildFleet(context.Background(), s, nil, time.Now(), 5*time.Minute) |
| 1163 | if err != nil { |
| 1164 | t.Fatal(err) |
| 1165 | } |
| 1166 | if f.ExpiredCanaries == nil || *f.ExpiredCanaries != 0 { |
| 1167 | t.Errorf("expired_canaries = %v, want 0 with no boxes", f.ExpiredCanaries) |
| 1168 | } |
| 1169 | } |
| 1170 | |
| 1171 | // TestFleetBoxCarriesExpiredCanaries keeps the per-box row honest too — the |
| 1172 | // fleet-wide number says something is expired somewhere, the boxes table |
| 1173 | // says which box, the same reasoning TestFleetBoxCarriesAdoption pins for |
| 1174 | // the adoption-timer reading. |
| 1175 | func TestFleetBoxCarriesExpiredCanaries(t *testing.T) { |
| 1176 | s := teststore(t) |
| 1177 | now := time.Now() |
| 1178 | three := 3 |
| 1179 | if err := status.WriteBox(context.Background(), s, |
| 1180 | status.CollectBox(nil, status.Probes{MachineID: "leaky", ExpiredCanaries: &three}, now)); err != nil { |
| 1181 | t.Fatal(err) |
| 1182 | } |
| 1183 | f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| 1184 | if err != nil { |
| 1185 | t.Fatal(err) |
| 1186 | } |
| 1187 | if len(f.Boxes) != 1 { |
| 1188 | t.Fatalf("boxes = %d, want 1", len(f.Boxes)) |
| 1189 | } |
| 1190 | if f.Boxes[0].ExpiredCanaries == nil || *f.Boxes[0].ExpiredCanaries != 3 { |
| 1191 | t.Errorf("boxes[0].expired_canaries = %v, want 3", f.Boxes[0].ExpiredCanaries) |
| 1192 | } |
| 1193 | } |
| 1194 | |
| 1195 | func TestFleetBoxCarriesAdoption(t *testing.T) { |
| 1196 | s := teststore(t) |
| 1197 | now := time.Now() |
| 1198 | no := false |
| 1199 | if err := status.WriteBox(context.Background(), s, |
| 1200 | status.CollectBox(nil, status.Probes{MachineID: "dead", AdoptRunning: &no}, now)); err != nil { |
| 1201 | t.Fatal(err) |
| 1202 | } |
| 1203 | f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| 1204 | if err != nil { |
| 1205 | t.Fatal(err) |
| 1206 | } |
| 1207 | if len(f.Boxes) != 1 { |
| 1208 | t.Fatalf("boxes = %d, want 1", len(f.Boxes)) |
| 1209 | } |
| 1210 | if f.Boxes[0].AdoptionRunning == nil || *f.Boxes[0].AdoptionRunning { |
| 1211 | t.Errorf("boxes[0].adoption_running = %v, want false — the row must name the box with the dead timer", |
| 1212 | f.Boxes[0].AdoptionRunning) |
| 1213 | } |
| 1214 | } |
| 1215 | |
| 1216 | // ---- Task 7: the exported box loader and the one staleness rule ------------- |
| 1217 | |
| 1218 | // TestLoadBoxesAndStaleness pins the two things the console's private host |
| 1219 | // detail and `carlos status -live` both read the bucket through: LoadBoxes |
| 1220 | // returns every snapshot WITH its private Metrics block intact (the merged |
| 1221 | // public feed drops it — this loader must not), and BoxStale answers the same |
| 1222 | // "older than 3 periods" question BuildFleet's own boxes table answers, so the |
| 1223 | // two renderings can never disagree about which host is dark. |
| 1224 | // oneFailingGet fails the read of one key and serves every other. |
| 1225 | type oneFailingGet struct { |
| 1226 | release.Store |
| 1227 | key string |
| 1228 | } |
| 1229 | |
| 1230 | func (f oneFailingGet) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 1231 | if key == f.key { |
| 1232 | return nil, "", errors.New("read timeout") |
| 1233 | } |
| 1234 | return f.Store.Get(ctx, key) |
| 1235 | } |
| 1236 | |
| 1237 | // LoadBoxesComplete returns what LoadBoxes returns and, separately, whether |
| 1238 | // every listed snapshot was read: a failed GET or a corrupt body is a |
| 1239 | // logged skip for LoadBoxes and an INCOMPLETE read here. The console's |
| 1240 | // automatic region placement fails closed on it (nearest-region re-review |
| 1241 | // r1): a box whose snapshot could not be read has unknown liveness, which |
| 1242 | // is not stale. |
| 1243 | func TestLoadBoxesCompleteReportsSkippedReads(t *testing.T) { |
| 1244 | ctx := context.Background() |
| 1245 | s := release.NewDirStore(t.TempDir()) |
| 1246 | for _, label := range []string{"alpha", "beta"} { |
| 1247 | if err := status.WriteBox(ctx, s, status.Box{Label: label, GeneratedAt: time.Now()}); err != nil { |
| 1248 | t.Fatal(err) |
| 1249 | } |
| 1250 | } |
| 1251 | boxes, complete, err := status.LoadBoxesComplete(ctx, s) |
| 1252 | if err != nil || !complete || len(boxes) != 2 { |
| 1253 | t.Fatalf("clean store: %d boxes, complete %v, err %v; want 2, true, nil", len(boxes), complete, err) |
| 1254 | } |
| 1255 | boxes, complete, err = status.LoadBoxesComplete(ctx, oneFailingGet{Store: s, key: "status/boxes/beta.json"}) |
| 1256 | if err != nil || complete || len(boxes) != 1 { |
| 1257 | t.Fatalf("one failed GET: %d boxes, complete %v, err %v; want 1 (alpha), false, nil", len(boxes), complete, err) |
| 1258 | } |
| 1259 | plain, err := status.LoadBoxes(ctx, oneFailingGet{Store: s, key: "status/boxes/beta.json"}) |
| 1260 | if err != nil || len(plain) != 1 { |
| 1261 | t.Fatalf("LoadBoxes under the same failure: %d boxes, err %v; want the unchanged logged-skip behaviour", len(plain), err) |
| 1262 | } |
| 1263 | _, etag, err := s.Get(ctx, "status/boxes/beta.json") |
| 1264 | if err != nil { |
| 1265 | t.Fatal(err) |
| 1266 | } |
| 1267 | if _, err := s.Put(ctx, "status/boxes/beta.json", []byte("{not json"), etag); err != nil { |
| 1268 | t.Fatal(err) |
| 1269 | } |
| 1270 | if boxes, complete, err := status.LoadBoxesComplete(ctx, s); err != nil || complete || len(boxes) != 1 { |
| 1271 | t.Fatalf("corrupt body: %d boxes, complete %v, err %v; want 1, false, nil", len(boxes), complete, err) |
| 1272 | } |
| 1273 | if _, complete, err := status.LoadBoxesComplete(ctx, release.NewDirStore(t.TempDir())); err != nil || !complete { |
| 1274 | t.Fatalf("empty prefix: complete %v, err %v; want true, nil", complete, err) |
| 1275 | } |
| 1276 | } |
| 1277 | |
| 1278 | func TestLoadBoxesAndStaleness(t *testing.T) { |
| 1279 | s := teststore(t) |
| 1280 | ctx := context.Background() |
| 1281 | now := time.Now() |
| 1282 | fresh := status.Box{GeneratedAt: now, Label: "ada-stargazing", |
| 1283 | Metrics: &status.BoxMetrics{Release: "abc1234", Tunnel: "connected"}} |
| 1284 | dark := status.Box{GeneratedAt: now.Add(-time.Hour), Label: "grace-kayaking"} |
| 1285 | for _, b := range []status.Box{fresh, dark} { |
| 1286 | if err := status.WriteBox(ctx, s, b); err != nil { |
| 1287 | t.Fatal(err) |
| 1288 | } |
| 1289 | } |
| 1290 | boxes, err := status.LoadBoxes(ctx, s) |
| 1291 | if err != nil || len(boxes) != 2 { |
| 1292 | t.Fatalf("LoadBoxes: %v, %d boxes", err, len(boxes)) |
| 1293 | } |
| 1294 | for _, b := range boxes { |
| 1295 | wantStale := b.Label == "grace-kayaking" |
| 1296 | if got := status.BoxStale(b, now, 5*time.Minute); got != wantStale { |
| 1297 | t.Fatalf("%s: stale = %v, want %v", b.Label, got, wantStale) |
| 1298 | } |
| 1299 | if b.Label == "ada-stargazing" { |
| 1300 | if b.Metrics == nil || b.Metrics.Release != "abc1234" || b.Metrics.Tunnel != "connected" { |
| 1301 | t.Fatalf("LoadBoxes dropped the private metrics block: %+v", b.Metrics) |
| 1302 | } |
| 1303 | } |
| 1304 | } |
| 1305 | } |
| 1306 | |
| 1307 | // TestBoxStaleMatchesFleetBoxStale is the anti-drift assertion: whatever |
| 1308 | // BoxStale says about a box must be exactly what BuildFleet publishes in that |
| 1309 | // box's FleetBox.Stale — one rule, one constant, two renderings. |
| 1310 | func TestBoxStaleMatchesFleetBoxStale(t *testing.T) { |
| 1311 | s := teststore(t) |
| 1312 | ctx := context.Background() |
| 1313 | now := time.Now() |
| 1314 | const period = 5 * time.Minute |
| 1315 | for _, b := range []status.Box{ |
| 1316 | boxFixture("just-now", "plenty", now), |
| 1317 | boxFixture("two-periods", "plenty", now.Add(-2*period)), |
| 1318 | boxFixture("exactly-three", "plenty", now.Add(-3*period)), |
| 1319 | boxFixture("long-dark", "plenty", now.Add(-time.Hour)), |
| 1320 | } { |
| 1321 | if err := status.WriteBox(ctx, s, b); err != nil { |
| 1322 | t.Fatal(err) |
| 1323 | } |
| 1324 | } |
| 1325 | f, err := status.BuildFleet(ctx, s, nil, now, period) |
| 1326 | if err != nil { |
| 1327 | t.Fatal(err) |
| 1328 | } |
| 1329 | boxes, err := status.LoadBoxes(ctx, s) |
| 1330 | if err != nil { |
| 1331 | t.Fatal(err) |
| 1332 | } |
| 1333 | byLabel := map[string]bool{} |
| 1334 | for _, fb := range f.Boxes { |
| 1335 | byLabel[fb.Label] = fb.Stale |
| 1336 | } |
| 1337 | for _, b := range boxes { |
| 1338 | if got, want := status.BoxStale(b, now, period), byLabel[b.Label]; got != want { |
| 1339 | t.Errorf("%s: BoxStale = %v, FleetBox.Stale = %v — the rule has drifted", b.Label, got, want) |
| 1340 | } |
| 1341 | } |
| 1342 | } |
| 1343 | |
| 1344 | // A first-route claim materializes a ROUTE, not an alias. Without its stamps |
| 1345 | // on the wire the console can never take it past "pending" — which is what |
| 1346 | // `carlos deploy -kind static -host <site>` has always shown. |
| 1347 | func TestCollectBoxCarriesRouteDNSStampsForNonPlatformRoutes(t *testing.T) { |
| 1348 | routes := []registry.Route{{ |
| 1349 | Host: "story.customer.com", Status: "ready", App: "site", |
| 1350 | DNSStatus: "pointing", DNSCheckedAt: 1756300000, CertExpiresAt: 1764000000, |
| 1351 | }} |
| 1352 | box := status.CollectBox(routes, status.Probes{}, time.Unix(1756300060, 0)) |
| 1353 | |
| 1354 | got, ok := box.Metrics.Aliases["story.customer.com"] |
| 1355 | if !ok { |
| 1356 | t.Fatal("a claim-backed route must publish a reading") |
| 1357 | } |
| 1358 | if got.DNSStatus != "pointing" || got.CertExpiresAt != 1764000000 { |
| 1359 | t.Fatalf("reading = %+v", got) |
| 1360 | } |
| 1361 | if got.Route != "story.customer.com" { |
| 1362 | t.Fatalf("Route = %q, want the host as its own primary (an empty claim route means exactly that)", got.Route) |
| 1363 | } |
| 1364 | if got.DNSCheckedAt != 1756300000 { |
| 1365 | t.Fatalf("DNSCheckedAt = %d, want the stamp's own age carried", got.DNSCheckedAt) |
| 1366 | } |
| 1367 | } |
| 1368 | |
| 1369 | // Platform routes are the deployment's own hosts, not custom domains, and |
| 1370 | // they already have RouteVersions. Keep the map to what the domains surface |
| 1371 | // actually joins against. |
| 1372 | func TestCollectBoxOmitsPlatformRoutesFromReadings(t *testing.T) { |
| 1373 | routes := []registry.Route{ |
| 1374 | {Host: "app.bdk.oncarlos.com", Status: "ready", Platform: true, DNSStatus: "pointing"}, |
| 1375 | // A custom domain alongside it, so Metrics attaches at all: Box.Metrics |
| 1376 | // is a POINTER and stays nil when nothing is known (snapshot.go:485-510). |
| 1377 | // Without this row the assertion below would nil-panic rather than |
| 1378 | // testing anything. |
| 1379 | {Host: "shop.customer.com", Status: "ready", DNSStatus: "pointing", DNSCheckedAt: 1756300000}, |
| 1380 | } |
| 1381 | box := status.CollectBox(routes, status.Probes{}, time.Unix(1756300060, 0)) |
| 1382 | if box.Metrics == nil { |
| 1383 | t.Fatal("Metrics must attach once a custom domain has a reading") |
| 1384 | } |
| 1385 | if _, ok := box.Metrics.Aliases["app.bdk.oncarlos.com"]; ok { |
| 1386 | t.Fatal("a platform route must not appear in the hostname readings") |
| 1387 | } |
| 1388 | if _, ok := box.Metrics.Aliases["shop.customer.com"]; !ok { |
| 1389 | t.Fatal("the custom domain must appear") |
| 1390 | } |
| 1391 | } |
| 1392 | |
| 1393 | // TestCollectBoxCarriesRouteDBBytes: every non-platform, non-static route |
| 1394 | // with a DBPath is sized through Probes.DBBytes into Metrics.RouteDBBytes. A |
| 1395 | // route the probe cannot size is absent rather than 0, and the hostname never |
| 1396 | // reaches the public fleet.json. |
| 1397 | func TestCollectBoxCarriesRouteDBBytes(t *testing.T) { |
| 1398 | routes := []registry.Route{ |
| 1399 | {Host: "sized.test", Kind: "instance", DBPath: "/data/sized.test.db"}, |
| 1400 | {Host: "missing.test", Kind: "instance", DBPath: "/data/missing.test.db"}, |
| 1401 | {Host: "nodb.test", Kind: "instance"}, |
| 1402 | {Host: "static.test", Kind: "static", DBPath: "/data/static.test.db"}, |
| 1403 | {Host: "platform.test", Kind: "instance", DBPath: "/data/sized.test.db", Platform: true}, |
| 1404 | } |
| 1405 | var asked []string |
| 1406 | probe := func(p string) *int64 { |
| 1407 | asked = append(asked, p) |
| 1408 | if p == "/data/sized.test.db" { |
| 1409 | n := int64(4096) |
| 1410 | return &n |
| 1411 | } |
| 1412 | return nil |
| 1413 | } |
| 1414 | b := status.CollectBox(routes, status.Probes{MachineID: "m", DBBytes: probe}, time.Now()) |
| 1415 | if b.Metrics == nil { |
| 1416 | t.Fatal("metrics block missing — RouteDBBytes should have attached it") |
| 1417 | } |
| 1418 | if len(b.Metrics.RouteDBBytes) != 1 || b.Metrics.RouteDBBytes["sized.test"] != 4096 { |
| 1419 | t.Fatalf("route_db_bytes = %v, want exactly {sized.test: 4096}", b.Metrics.RouteDBBytes) |
| 1420 | } |
| 1421 | if strings.Join(asked, ",") != "/data/sized.test.db,/data/missing.test.db" { |
| 1422 | t.Errorf("probed %v, want only the two instance routes with a DBPath", asked) |
| 1423 | } |
| 1424 | |
| 1425 | s := teststore(t) |
| 1426 | ctx := context.Background() |
| 1427 | if err := status.WriteBox(ctx, s, b); err != nil { |
| 1428 | t.Fatal(err) |
| 1429 | } |
| 1430 | f, err := status.BuildFleet(ctx, s, routes, time.Now(), 5*time.Minute) |
| 1431 | if err != nil { |
| 1432 | t.Fatal(err) |
| 1433 | } |
| 1434 | fj, err := json.Marshal(f) |
| 1435 | if err != nil { |
| 1436 | t.Fatal(err) |
| 1437 | } |
| 1438 | if strings.Contains(string(fj), "sized.test") { |
| 1439 | t.Errorf("fleet.json leaked a route_db_bytes hostname: %s", fj) |
| 1440 | } |
| 1441 | |
| 1442 | // No probe at all: nothing sized, and no Metrics block from this alone. |
| 1443 | if b := status.CollectBox(routes[:1], status.Probes{MachineID: "m"}, time.Now()); b.Metrics != nil { |
| 1444 | t.Errorf("metrics = %+v with no DBBytes probe, want absent", b.Metrics) |
| 1445 | } |
| 1446 | } |
| 1447 | |
| 1448 | // TestCollectBoxCarriesConfigRefusals (amadan #213): a box that refuses to |
| 1449 | // take an instance because it cannot provide that instance's config says so |
| 1450 | // in its private Metrics block, where an operator can see which hosts and |
| 1451 | // why. The hostnames never reach the public fleet.json, and a box with no |
| 1452 | // refusal carries no key at all. |
| 1453 | func TestCollectBoxCarriesConfigRefusals(t *testing.T) { |
| 1454 | since := time.Date(2026, 10, 5, 9, 0, 0, 0, time.UTC) |
| 1455 | refusals := []status.ConfigRefusal{ |
| 1456 | {Host: "refused-secret.example.com", Reason: "no_key", Since: since, Last: since.Add(time.Minute), Count: 3}, |
| 1457 | } |
| 1458 | routes := []registry.Route{{Host: "refused-secret.example.com", Kind: "instance", Hibernate: true}} |
| 1459 | b := status.CollectBox(routes, status.Probes{MachineID: "m", ConfigRefusals: refusals, ConfigRefusalsTotal: 5}, time.Now()) |
| 1460 | if b.Metrics == nil { |
| 1461 | t.Fatal("metrics block missing: a refusal alone should attach it") |
| 1462 | } |
| 1463 | if got := b.Metrics.ConfigRefusals; len(got) != 1 || got[0] != refusals[0] { |
| 1464 | t.Fatalf("config_refusals = %+v, want %+v", got, refusals) |
| 1465 | } |
| 1466 | if b.Metrics.ConfigRefusalsTotal != 5 { |
| 1467 | t.Errorf("config_refusals_total = %d, want 5", b.Metrics.ConfigRefusalsTotal) |
| 1468 | } |
| 1469 | |
| 1470 | s := teststore(t) |
| 1471 | ctx := context.Background() |
| 1472 | if err := status.WriteBox(ctx, s, b); err != nil { |
| 1473 | t.Fatal(err) |
| 1474 | } |
| 1475 | f, err := status.BuildFleet(ctx, s, routes, time.Now(), 5*time.Minute) |
| 1476 | if err != nil { |
| 1477 | t.Fatal(err) |
| 1478 | } |
| 1479 | fj, err := json.Marshal(f) |
| 1480 | if err != nil { |
| 1481 | t.Fatal(err) |
| 1482 | } |
| 1483 | for _, needle := range []string{"refused-secret.example.com", "no_key", "config_refusals"} { |
| 1484 | if strings.Contains(string(fj), needle) { |
| 1485 | t.Errorf("fleet.json leaked %q: %s", needle, fj) |
| 1486 | } |
| 1487 | } |
| 1488 | |
| 1489 | // Nothing refused: no Metrics block from this alone, no key on the wire. |
| 1490 | quiet := status.CollectBox(routes, status.Probes{MachineID: "m"}, time.Now()) |
| 1491 | if quiet.Metrics != nil { |
| 1492 | t.Errorf("metrics = %+v with no refusal, want absent", quiet.Metrics) |
| 1493 | } |
| 1494 | raw, _ := json.Marshal(status.CollectBox(routes, status.Probes{MachineID: "m", MemTotalBytes: new(int64)}, time.Now())) |
| 1495 | if strings.Contains(string(raw), "config_refusals") { |
| 1496 | t.Errorf("a box with no refusal publishes config_refusals: %s", raw) |
| 1497 | } |
| 1498 | } |
| 1499 | |
| 1500 | // TestCollectBoxCarriesReleaseRefusals (D1): a box that rolled a build back |
| 1501 | // says so in its private Metrics block, naming the host, version and reason. |
| 1502 | // The hostname never reaches the public fleet.json, and a box with no |
| 1503 | // refusal carries no key at all. |
| 1504 | func TestCollectBoxCarriesReleaseRefusals(t *testing.T) { |
| 1505 | at := time.Date(2026, 10, 6, 9, 30, 40, 0, time.UTC) |
| 1506 | refusals := []status.ReleaseRefusal{ |
| 1507 | {Host: "rolled-back.example.com", Version: "v2-secret-build", Reason: "exited: status=2", RefusedAt: at}, |
| 1508 | } |
| 1509 | routes := []registry.Route{{Host: "rolled-back.example.com", Kind: "instance", Hibernate: true}} |
| 1510 | b := status.CollectBox(routes, status.Probes{MachineID: "m", ReleaseRefusals: refusals}, time.Now()) |
| 1511 | if b.Metrics == nil { |
| 1512 | t.Fatal("metrics block missing: a release refusal alone should attach it") |
| 1513 | } |
| 1514 | if got := b.Metrics.ReleaseRefusals; len(got) != 1 || got[0] != refusals[0] { |
| 1515 | t.Fatalf("release_refusals = %+v, want %+v", got, refusals) |
| 1516 | } |
| 1517 | |
| 1518 | s := teststore(t) |
| 1519 | ctx := context.Background() |
| 1520 | if err := status.WriteBox(ctx, s, b); err != nil { |
| 1521 | t.Fatal(err) |
| 1522 | } |
| 1523 | f, err := status.BuildFleet(ctx, s, routes, time.Now(), 5*time.Minute) |
| 1524 | if err != nil { |
| 1525 | t.Fatal(err) |
| 1526 | } |
| 1527 | fj, err := json.Marshal(f) |
| 1528 | if err != nil { |
| 1529 | t.Fatal(err) |
| 1530 | } |
| 1531 | for _, needle := range []string{"rolled-back.example.com", "v2-secret-build", "exited: status=2", "release_refusals"} { |
| 1532 | if strings.Contains(string(fj), needle) { |
| 1533 | t.Errorf("fleet.json leaked %q: %s", needle, fj) |
| 1534 | } |
| 1535 | } |
| 1536 | |
| 1537 | quiet := status.CollectBox(routes, status.Probes{MachineID: "m"}, time.Now()) |
| 1538 | if quiet.Metrics != nil { |
| 1539 | t.Errorf("metrics = %+v with no refusal, want absent", quiet.Metrics) |
| 1540 | } |
| 1541 | raw, _ := json.Marshal(status.CollectBox(routes, status.Probes{MachineID: "m", MemTotalBytes: new(int64)}, time.Now())) |
| 1542 | if strings.Contains(string(raw), "release_refusals") { |
| 1543 | t.Errorf("a box with no refusal publishes release_refusals: %s", raw) |
| 1544 | } |
| 1545 | } |
| 1546 | |
| 1547 | // TestCollectBoxCarriesProofWaits (D3): a box holding a host on its current |
| 1548 | // build for a dev proof says so in its private Metrics block. The hostname |
| 1549 | // and build never reach the public fleet.json, and a box with no wait carries |
| 1550 | // no key at all. |
| 1551 | func TestCollectBoxCarriesProofWaits(t *testing.T) { |
| 1552 | at := time.Date(2026, 10, 6, 9, 30, 40, 0, time.UTC) |
| 1553 | waits := []status.ProofWait{ |
| 1554 | {Host: "waiting.example.com", Version: "v3-secret-build", SHA256: "e47fcc1e0123456789abcdef", Prover: "dev", |
| 1555 | Reason: "no proof from dev", Since: at}, |
| 1556 | } |
| 1557 | routes := []registry.Route{{Host: "waiting.example.com", Kind: "instance", Hibernate: true}} |
| 1558 | b := status.CollectBox(routes, status.Probes{MachineID: "m", ProofWaits: waits}, time.Now()) |
| 1559 | if b.Metrics == nil { |
| 1560 | t.Fatal("metrics block missing: a proof wait alone should attach it") |
| 1561 | } |
| 1562 | if got := b.Metrics.ProofWaits; len(got) != 1 || got[0] != waits[0] { |
| 1563 | t.Fatalf("proof_waits = %+v, want %+v", got, waits) |
| 1564 | } |
| 1565 | |
| 1566 | s := teststore(t) |
| 1567 | ctx := context.Background() |
| 1568 | if err := status.WriteBox(ctx, s, b); err != nil { |
| 1569 | t.Fatal(err) |
| 1570 | } |
| 1571 | f, err := status.BuildFleet(ctx, s, routes, time.Now(), 5*time.Minute) |
| 1572 | if err != nil { |
| 1573 | t.Fatal(err) |
| 1574 | } |
| 1575 | fj, err := json.Marshal(f) |
| 1576 | if err != nil { |
| 1577 | t.Fatal(err) |
| 1578 | } |
| 1579 | for _, needle := range []string{"waiting.example.com", "v3-secret-build", "e47fcc1e", "no proof from dev", "proof_waits"} { |
| 1580 | if strings.Contains(string(fj), needle) { |
| 1581 | t.Errorf("fleet.json leaked %q: %s", needle, fj) |
| 1582 | } |
| 1583 | } |
| 1584 | |
| 1585 | quiet := status.CollectBox(routes, status.Probes{MachineID: "m"}, time.Now()) |
| 1586 | if quiet.Metrics != nil { |
| 1587 | t.Errorf("metrics = %+v with no wait, want absent", quiet.Metrics) |
| 1588 | } |
| 1589 | raw, _ := json.Marshal(status.CollectBox(routes, status.Probes{MachineID: "m", MemTotalBytes: new(int64)}, time.Now())) |
| 1590 | if strings.Contains(string(raw), "proof_waits") { |
| 1591 | t.Errorf("a box with no wait publishes proof_waits: %s", raw) |
| 1592 | } |
| 1593 | } |
| 1594 | |
| 1595 | // The config guard's bypass is visible in the private block, and only there. |
| 1596 | func TestCollectBoxCarriesTheConfigGuardBypass(t *testing.T) { |
| 1597 | b := status.CollectBox(nil, status.Probes{MachineID: "m", ConfigGuard: "bypassed (CARLOS_NO_SEALED_CONFIG)"}, time.Now()) |
| 1598 | if b.Metrics == nil || b.Metrics.ConfigGuard != "bypassed (CARLOS_NO_SEALED_CONFIG)" { |
| 1599 | t.Fatalf("metrics = %+v, want the bypass word", b.Metrics) |
| 1600 | } |
| 1601 | s := teststore(t) |
| 1602 | ctx := context.Background() |
| 1603 | if err := status.WriteBox(ctx, s, b); err != nil { |
| 1604 | t.Fatal(err) |
| 1605 | } |
| 1606 | f, err := status.BuildFleet(ctx, s, nil, time.Now(), 5*time.Minute) |
| 1607 | if err != nil { |
| 1608 | t.Fatal(err) |
| 1609 | } |
| 1610 | fj, _ := json.Marshal(f) |
| 1611 | if strings.Contains(string(fj), "bypass") || strings.Contains(string(fj), "NO_SEALED") { |
| 1612 | t.Errorf("fleet.json carries the bypass: %s", fj) |
| 1613 | } |
| 1614 | if quiet := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Now()); quiet.Metrics != nil { |
| 1615 | t.Errorf("a box in the normal mode attached metrics: %+v", quiet.Metrics) |
| 1616 | } |
| 1617 | } |
| 1618 | |
| 1619 | // TestCollectBoxCarriesBuildStates (settings spec rev 11 §4.4): a box names |
| 1620 | // the build-state builds of the instances it hosts in its private Metrics |
| 1621 | // block, so the housekeeper can protect them. The hostnames and builds never |
| 1622 | // reach the public fleet.json, and a box with none carries no key at all. |
| 1623 | func TestCollectBoxCarriesBuildStates(t *testing.T) { |
| 1624 | pins := map[string]status.BuildStatePins{ |
| 1625 | "held-secret.example.com": {Account: "bab", App: "km", Builds: []string{"v1-secret-build", "v2-secret-build"}, Complete: true}, |
| 1626 | } |
| 1627 | routes := []registry.Route{{Host: "held-secret.example.com", Kind: "instance", Hibernate: true}} |
| 1628 | b := status.CollectBox(routes, status.Probes{MachineID: "m", BuildStates: pins}, time.Now()) |
| 1629 | if b.Metrics == nil { |
| 1630 | t.Fatal("metrics block missing: build states alone should attach it") |
| 1631 | } |
| 1632 | if got := b.Metrics.BuildStates; !reflect.DeepEqual(got, pins) { |
| 1633 | t.Fatalf("build_states = %+v, want %+v", got, pins) |
| 1634 | } |
| 1635 | |
| 1636 | s := teststore(t) |
| 1637 | ctx := context.Background() |
| 1638 | if err := status.WriteBox(ctx, s, b); err != nil { |
| 1639 | t.Fatal(err) |
| 1640 | } |
| 1641 | boxes, complete, err := status.LoadBoxesComplete(ctx, s) |
| 1642 | if err != nil || !complete || len(boxes) != 1 || boxes[0].Metrics == nil || !reflect.DeepEqual(boxes[0].Metrics.BuildStates, pins) { |
| 1643 | t.Fatalf("read back %+v, %v, %v; want the build states", boxes, complete, err) |
| 1644 | } |
| 1645 | f, err := status.BuildFleet(ctx, s, routes, time.Now(), 5*time.Minute) |
| 1646 | if err != nil { |
| 1647 | t.Fatal(err) |
| 1648 | } |
| 1649 | fj, err := json.Marshal(f) |
| 1650 | if err != nil { |
| 1651 | t.Fatal(err) |
| 1652 | } |
| 1653 | for _, needle := range []string{"held-secret.example.com", "v1-secret-build", "build_states"} { |
| 1654 | if strings.Contains(string(fj), needle) { |
| 1655 | t.Errorf("fleet.json leaked %q: %s", needle, fj) |
| 1656 | } |
| 1657 | } |
| 1658 | |
| 1659 | raw, _ := json.Marshal(status.CollectBox(routes, status.Probes{MachineID: "m", MemTotalBytes: new(int64)}, time.Now())) |
| 1660 | if strings.Contains(string(raw), "build_states") { |
| 1661 | t.Errorf("a box with no build states publishes build_states: %s", raw) |
| 1662 | } |
| 1663 | } |
| 1664 | |
| 1665 | // A box that could not collect its build states says so (Astra I2 #5): the |
| 1666 | // error alone attaches the Metrics block, so the housekeeper reads "could not |
| 1667 | // collect", never an absent map it might take for "nothing to protect". |
| 1668 | func TestCollectBoxCarriesABuildStatesFailure(t *testing.T) { |
| 1669 | b := status.CollectBox(nil, status.Probes{MachineID: "m", BuildStatesErr: "listing routes for build states: database is locked"}, time.Now()) |
| 1670 | if b.Metrics == nil || b.Metrics.BuildStatesErr != "listing routes for build states: database is locked" { |
| 1671 | t.Fatalf("metrics = %+v, want the collection failure", b.Metrics) |
| 1672 | } |
| 1673 | raw, _ := json.Marshal(b) |
| 1674 | if !strings.Contains(string(raw), `"build_states_err"`) { |
| 1675 | t.Fatalf("the failure is not published: %s", raw) |
| 1676 | } |
| 1677 | } |
| 1678 | |
| 1679 | // TestCollectBoxCarriesTheUpdateSettingsProtocolAndDecisions (settings spec |
| 1680 | // §4.3, §4.4): a box that enforces update settings says so, and names root's |
| 1681 | // last decision per instance in the private Metrics block. The protocol alone |
| 1682 | // attaches the block (the console reads it to lift its refusal); the hosts |
| 1683 | // and builds never reach the public fleet.json. |
| 1684 | func TestCollectBoxCarriesTheUpdateSettingsProtocolAndDecisions(t *testing.T) { |
| 1685 | routes := []registry.Route{{Host: "decided-secret.example.com", Kind: "instance", Hibernate: true}} |
| 1686 | only := status.CollectBox(routes, status.Probes{MachineID: "m", UpdateSettings: status.UpdateSettingsProtocol}, time.Now()) |
| 1687 | if only.Metrics == nil || only.Metrics.UpdateSettings != status.UpdateSettingsProtocol { |
| 1688 | t.Fatalf("metrics = %+v, want the protocol alone to attach the block", only.Metrics) |
| 1689 | } |
| 1690 | raw, _ := json.Marshal(only) |
| 1691 | if strings.Contains(string(raw), `"updates"`) { |
| 1692 | t.Errorf("a box with no decisions publishes updates: %s", raw) |
| 1693 | } |
| 1694 | |
| 1695 | two := 2 |
| 1696 | ups := map[string]status.HostUpdates{ |
| 1697 | "decided-secret.example.com": {Desired: "v-secret-desired", Reason: "major", Detail: "v1", Baking: "v-secret-baking", Major: &two, Pending: 3, Completed: 2, CompletedFailed: true, |
| 1698 | Account: "acct-secret-tenant", App: "app-secret-tenant"}, |
| 1699 | } |
| 1700 | b := status.CollectBox(routes, status.Probes{MachineID: "m", Updates: ups}, time.Now()) |
| 1701 | if b.Metrics == nil || !reflect.DeepEqual(b.Metrics.Updates, ups) { |
| 1702 | t.Fatalf("updates = %+v, want %+v", b.Metrics, ups) |
| 1703 | } |
| 1704 | s := teststore(t) |
| 1705 | ctx := context.Background() |
| 1706 | if err := status.WriteBox(ctx, s, b); err != nil { |
| 1707 | t.Fatal(err) |
| 1708 | } |
| 1709 | boxes, complete, err := status.LoadBoxesComplete(ctx, s) |
| 1710 | if err != nil || !complete || len(boxes) != 1 || boxes[0].Metrics == nil || !reflect.DeepEqual(boxes[0].Metrics.Updates, ups) { |
| 1711 | t.Fatalf("read back %+v, %v, %v; want the decisions", boxes, complete, err) |
| 1712 | } |
| 1713 | f, err := status.BuildFleet(ctx, s, routes, time.Now(), 5*time.Minute) |
| 1714 | if err != nil { |
| 1715 | t.Fatal(err) |
| 1716 | } |
| 1717 | fj, _ := json.Marshal(f) |
| 1718 | for _, needle := range []string{"decided-secret.example.com", "v-secret-desired", "v-secret-baking", "acct-secret-tenant", "app-secret-tenant", `"updates"`, "update_settings"} { |
| 1719 | if strings.Contains(string(fj), needle) { |
| 1720 | t.Errorf("fleet.json leaked %q: %s", needle, fj) |
| 1721 | } |
| 1722 | } |
| 1723 | } |
| 1724 | |
amadan