carlos / platform Public

Clone
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package status_test
2
3import (
4 "context"
5 "encoding/json"
6 "strings"
7 "testing"
8 "time"
9
10 "github.com/carlosframework/platform/internal/registry"
11 "github.com/carlosframework/platform/internal/status"
12)
13
14// ---- Task 5: Contract tests — allowlist + digit scan ----
15// The allowlist IS the schema. Adding a field to the structs without adding
16// it here fails the build — that friction is the point: every new field is a
17// deliberate publication decision. This file guards the public JSON contract
18// that the fleet status page publishes, ensuring no accidental leaks of
19// hostnames, IDs, or other sensitive data slip into the output.
20
21// allowedBoxKeys are THE schema for Box's JSON output. Any new field in Box
22// must be explicitly added here to pass the test — design friction that makes
23// the publication decision deliberate, never accidental. The map keys are path
24// strings ("" for root, "fieldname" for nested, "fieldname[]" for array
25// elements); values are the allowed field names at that path.
26var allowedBoxKeys = map[string][]string{
27 // "adoption_running" is a deliberate publication (pipelines spec §5): it
28 // says whether the platform's own release-adoption timer is armed on this
29 // box. It carries no tenant, hostname or machine identity — only whether
30 // an opaquely-labelled box is picking up its own releases.
31 //
32 // "expired_canaries" is likewise deliberate (pipelines spec §5 "Canaries
33 // expire"): a COUNT of this box's own canary/* routes past their TTL,
34 // never which ones — no hostname, no channel name, no tenant identity.
35 //
36 // "config_path_running" is the issue #44 sibling of adoption_running: it
37 // says whether this box's config-sync trigger unit (carlos-adopt.path) is
38 // armed, i.e. whether config reaches unit-backed routes in seconds or only
39 // on the hourly floor. Same publication argument — a platform-own unit's
40 // liveness on an opaquely-labelled box, no tenant in it anywhere.
41 //
42 // "restart_path_running" is the issue #113 sibling of both: whether this
43 // box's restart trigger unit (carlos-restart.path) is armed, i.e. whether a
44 // member's `carlos restart` reaches its units in seconds or waits out the
45 // hourly floor. Identical publication argument — one platform-own unit's
46 // liveness on an opaquely-labelled box, and no tenant anywhere in it (it
47 // says nothing about WHICH app was restarted, or that any was).
48 //
49 // "metrics" is the one key in this allowlist that is EXEMPT from the
50 // band-word-not-a-number rule every other Box field follows (Task 5,
51 // internal/status/snapshot.go's package doc): it carries raw byte
52 // counts, a load average, a release sha, and tunnel state, unbanded.
53 // That's safe only because Box (and this "metrics" key with it) never
54 // leaves the bucket — status/boxes/<label>.json is read by fleet-status
55 // machinery and operators, never served publicly (internal/edge/status.go's
56 // fixed three-object menu doesn't include it) — and because
57 // allowedFleetKeys below has no "metrics" entry: BuildFleet's explicit
58 // field-by-field copy into FleetBox never names it, so TestFleetSchemaAllowlisted
59 // would fail the moment that ever changed.
60 //
61 // "metrics.route_versions" (Task 8, issue #78) carries real hostnames —
62 // the one field under "metrics" that isn't just an unbanded number —
63 // safe for exactly the same bucket-private reason: BuildFleet's
64 // field-by-field copy never names "metrics" at all, so no hostname here
65 // can reach FleetBox or fleet.json regardless of what this map holds.
66 // "metrics.upstreams" (multi-upstream plan, Task 6) is route_versions'
67 // sibling and then some: its KEYS are route hostnames and its VALUES carry
68 // real backend addresses. It is safe on the identical structural ground —
69 // BuildFleet's field-by-field copy never names "metrics", so nothing under
70 // it can reach FleetBox or fleet.json — and it is spelled out to one more
71 // level here than route_versions needs to be, because its values are
72 // objects rather than plain strings: "metrics.upstreams.*[]" pins the
73 // per-upstream fields, so a future field on UpstreamState is a deliberate
74 // publication decision too, not a silent ride-along.
75 // "metrics.layout" (Task 5, isolated-exec plan 4) is the box's layout
76 // report: which mode it last converged to, and — hostname-keyed, same
77 // reasoning as metrics.route_versions and metrics.upstreams just above —
78 // every instance-kind route's word from the layout pass's own Classify.
79 // Safe on the identical structural ground: BuildFleet's field-by-field
80 // copy never names "metrics", so no hostname here can reach FleetBox or
81 // fleet.json regardless of what this map holds.
82 "": {"generated_at", "label", "region", "routes", "load", "capacity", "replication", "adoption_running", "config_path_running", "restart_path_running", "expired_canaries", "metrics"},
83 "routes": {"static", "resident", "hibernating", "awake", "asleep"},
84 "replication": {"healthy", "lag_seconds"},
85 "metrics": {"mem_total_bytes", "mem_avail_bytes", "load1", "disk_free_bytes", "release", "tunnel", "route_versions", "upstreams", "layout", "config_refusals", "config_refusals_total", "release_refusals", "proof_waits", "config_guard", "build_states", "build_states_err", "update_settings", "updates"},
86 "metrics.route_versions": {wildcardAnyKey},
87 "metrics.upstreams": {wildcardAnyKey},
88 "metrics.upstreams.*[]": {"addr", "up", "since", "fails", "last_error"},
89 "metrics.layout": {"mode", "rows", "default", "default_blocked"},
90 "metrics.layout.rows": {wildcardAnyKey},
91 "metrics.layout.rows.*": {"layout", "reason", "note"},
92 // config_refusals (amadan #213) names hosts and a closed reason word, in
93 // the private block only. Pinned to the field level so a future field on
94 // the refusal is a publication decision too.
95 "metrics.config_refusals[]": {"host", "reason", "since", "last", "count"},
96 // release_refusals (D1) names the host and build the box rolled back, in
97 // the private block only, pinned to the field level for the same reason.
98 "metrics.release_refusals[]": {"host", "version", "reason", "refused_at", "kind", "recovery_error"},
99 // proof_waits (D3) names the host and build the box is holding for a dev
100 // proof, in the private block only, pinned to the field level too.
101 "metrics.proof_waits[]": {"host", "version", "sha256", "prover", "reason", "since"},
102 // build_states (settings spec rev 11 §4.4) is host -> the builds that
103 // host's build state names, in the private block only, for the
104 // housekeeper's retention. Keys are hostnames; the values are pinned to
105 // the field level.
106 "metrics.build_states": {wildcardAnyKey},
107 "metrics.build_states.*": {"account", "app", "builds", "complete"},
108 // updates (settings spec §4.3) is host -> root's last decision for that
109 // host, in the private block only; the values are pinned to the field level.
110 // account and app are the host's tenant (Astra r1, R18), as build_states
111 // carries it.
112 "metrics.updates": {wildcardAnyKey},
113 "metrics.updates.*": {"desired", "reason", "detail", "baking", "major", "pending", "completed", "completed_failed", "account", "app"},
114}
115
116// allowedFleetKeys are THE schema for Fleet's JSON output. Same friction as
117// allowedBoxKeys: every new field is a deliberate publication decision.
118var allowedFleetKeys = map[string][]string{
119 "": {"generated_at", "boxes", "instances", "capacity", "apps", "apps_unlisted", "services", "replication", "adoption_running", "expired_canaries"},
120 "boxes[]": {"label", "region", "load", "stale", "routes", "adoption_running", "config_path_running", "restart_path_running", "expired_canaries"},
121 "boxes[].routes": {"static", "resident", "hibernating", "awake", "asleep"},
122 "instances": {"total", "static", "resident", "hibernating", "awake", "asleep"},
123 "capacity": {"level"},
124 "apps[]": {"name", "instances", "healthy", "channel", "version"},
125 "services[]": {"name", "up"},
126 "replication": {"healthy", "lag_seconds"},
127}
128
129// assertOnlyKeys walks the decoded JSON map recursively, ensuring every key
130// at every path is in the allowlist for that path. Arrays walk their elements;
131// a key not in the allowlist fails with the offending path. This is the guard:
132// if a new field leaks into the JSON without an explicit allowlist entry, the
133// test fails and forces a review.
134func assertOnlyKeys(t *testing.T, j []byte, allowedKeys map[string][]string) {
135 t.Helper()
136 var decoded interface{}
137 if err := json.Unmarshal(j, &decoded); err != nil {
138 t.Fatalf("unmarshal JSON: %v", err)
139 }
140 walkKeys(t, decoded, "", allowedKeys)
141}
142
143// wildcardAnyKey marks a path (in an allowedKeys map) whose object keys are
144// DATA, not a fixed field-name schema — metrics.route_versions (Task 8,
145// issue #78) is host -> Version and metrics.upstreams (multi-upstream plan,
146// Task 6) is host -> []UpstreamState, so their keys are hostnames chosen at
147// runtime, which no fixed allowlist could name in advance. Values still
148// recurse normally, so this only relaxes the KEY-NAME check, never the shape
149// check: a wildcarded map's values walk on under the literal path segment
150// "*" (walkKeys, below), which is what lets "metrics.upstreams.*[]" pin the
151// fields of the objects inside — a runtime-chosen key must not also buy its
152// value an unexamined shape.
153const wildcardAnyKey = "*"
154
155// walkKeys recursively traverses the decoded JSON structure, checking that
156// every key at every path is in the allowlist. It handles maps and arrays
157// appropriately, using "" for root, "fieldname" for nested maps, and
158// "fieldname[]" for array element paths.
159func walkKeys(t *testing.T, obj interface{}, path string, allowedKeys map[string][]string) {
160 switch v := obj.(type) {
161 case map[string]interface{}:
162 // This is a map (object). Check that every key is in the allowlist
163 // for this path — unless the path is wildcarded (see wildcardAnyKey),
164 // in which case the keys themselves are data.
165 allowed := allowedKeys[path]
166 wildcard := len(allowed) == 1 && allowed[0] == wildcardAnyKey
167 for key := range v {
168 if !wildcard {
169 found := false
170 for _, a := range allowed {
171 if a == key {
172 found = true
173 break
174 }
175 }
176 if !found {
177 t.Errorf("unexpected key %q at path %q", key, path)
178 continue
179 }
180 }
181 // Recurse into this key's value, building the child path. Under a
182 // wildcard the key itself is data (a hostname), so every value
183 // walks under the same literal "*" segment rather than a path
184 // only that one runtime key would ever match — otherwise the
185 // values of a wildcarded map could never be allowlisted at all,
186 // and any shape inside them would pass unchecked.
187 var childPath string
188 switch {
189 case wildcard:
190 childPath = path + ".*"
191 case path == "":
192 childPath = key
193 default:
194 childPath = path + "." + key
195 }
196 walkKeys(t, v[key], childPath, allowedKeys)
197 }
198
199 case []interface{}:
200 // This is an array. Recurse into each element with path + "[]".
201 elemPath := path + "[]"
202 for _, elem := range v {
203 walkKeys(t, elem, elemPath, allowedKeys)
204 }
205
206 case string, float64, bool, nil:
207 // Leaf values; nothing to check.
208 }
209}
210
211// scanForLeaks searches the JSON bytes for any of the banned strings and
212// returns the list of banned strings actually found. Used by tests to verify
213// sensitive data (hostnames, IDs) never leaks into the output. Delegates to
214// status.ScanForLeaks (dryrun.go) rather than reimplementing the scan here:
215// DryRun needs this same check at runtime (this file is package
216// status_test, which DryRun — living in package status — cannot import), and
217// two copies of "search bytes for banned substrings" would risk silently
218// drifting into checking different things.
219func scanForLeaks(j []byte, banned []string) []string {
220 return status.ScanForLeaks(j, banned)
221}
222
223// fixtureRoutes creates test routes with a variety of kinds, some hibernating,
224// and at least one opted-in app. The result is suitable for CollectBox and
225// BuildFleet tests. It includes:
226// - A static route (not hibernating, not instance)
227// - A resident route (not hibernating, instance kind)
228// - A hibernating route with an opted-in app
229// - Additional hibernating routes without opt-in to test the various counts
230func fixtureRoutes() []registry.Route {
231 return []registry.Route{
232 {
233 Host: "static-route.internal",
234 Kind: "static",
235 App: "",
236 Status: "ready",
237 Socket: "",
238 Hibernate: false,
239 },
240 {
241 Host: "resident-app.internal",
242 Kind: "instance",
243 App: "resident-service",
244 Status: "ready",
245 Socket: "",
246 Hibernate: false,
247 StatusNamed: false,
248 },
249 {
250 Host: "hibernating-opted.internal",
251 Kind: "instance",
252 App: "opted-app",
253 Status: "ready",
254 Socket: "/var/run/opted-app.sock",
255 Hibernate: true,
256 StatusNamed: true,
257 Channel: "stable",
258 Version: "abc1234",
259 },
260 {
261 Host: "hibernating-not-opted.internal",
262 Kind: "instance",
263 App: "not-opted",
264 Status: "ready",
265 Socket: "/var/run/not-opted.sock",
266 Hibernate: true,
267 StatusNamed: false,
268 },
269 {
270 Host: "svc-pdf.internal",
271 Kind: "service",
272 App: "svc/pdf",
273 Status: "ready",
274 Socket: "",
275 Hibernate: false,
276 },
277 }
278}
279
280// fixtureProbes creates test probes with known fractions so that banding
281// produces non-nil results. It uses values in the middle bands so that
282// load and capacity both resolve to band words (not nil).
283//
284// The six Metrics fields (memTotal/memAvail/load1/diskFree/release/tunnel)
285// are populated here too, with values DISTINCT from dryrun.go's own
286// dryRunMetricsNeedles fixtures — the two harnesses check different things
287// (this file's allowlist tests guard Box/Fleet's declared JSON schema;
288// dryrun.go's leak scan guards the real publish pipeline end to end) and
289// must never share fixture values, or a bug that only one harness would
290// catch could hide behind the other's coincidentally-matching needle.
291func fixtureProbes() status.Probes {
292 memFrac := 0.75 // Between 0.70 and 0.90 -> "busy"
293 diskFrac := 0.80 // Between 0.70 and 0.90 -> "getting-full"
294 lagSecs := int64(300) // 5 minutes, healthy (< 15 min)
295 adopting := true // a real reading, so the field is scanned with a value in it
296 contractMajor := 2
297
298 memTotal := int64(8123456789) // distinct from dryrun.go's 4242424242
299 memAvail := int64(3123456789) // distinct from dryrun.go's 4242424241
300 diskFree := int64(1123456789) // distinct from dryrun.go's 4242424240
301 load1 := 1.23 // distinct from dryrun.go's 42.42
302
303 // Listening function returns true for hibernating routes, false otherwise.
304 listening := func(sock string) *bool {
305 if strings.HasPrefix(sock, "/var/run/") {
306 // Hibernating routes; return true for "awake"
307 v := strings.Contains(sock, "opted")
308 return &v
309 }
310 return nil
311 }
312
313 // One probed route, so metrics.upstreams is actually exercised by the
314 // allowlist walk rather than merely permitted by it — an unexercised
315 // schema entry guards nothing. Distinct from dryrun.go's own upstream
316 // fixture for the same reason every value here is (see the doc comment).
317 upstreams := map[string][]status.UpstreamState{
318 "contract-fixture.internal": {
319 {Addr: "10.31.31.11:9090", Up: true, Since: time.Now().Add(-time.Hour)},
320 {Addr: "10.31.31.12:9090", Up: false, Since: time.Now().Add(-time.Minute), Fails: 5, LastError: "status 500 (want 200)"},
321 },
322 }
323
324 // One flat row and one foreign row, so both the bare "layout" field
325 // (mode) and the wildcarded, host-keyed "rows" map — including a row
326 // that carries a reason — are actually exercised by the allowlist walk,
327 // the same reasoning fixtureProbes' own comment gives for upstreams.
328 layoutReport := &status.LayoutReport{
329 Mode: "flat",
330 Rows: map[string]status.RowLayout{
331 "contract-layout.internal": {Layout: "flat"},
332 "contract-foreign.internal": {Layout: "foreign",
333 Reason: `db "x" socket "y" are neither the flat nor the nested defaults`},
334 },
335 }
336
337 return status.Probes{
338 MachineID: "test-machine-id",
339 Region: "us-west-1",
340 Upstreams: upstreams,
341 Layout: layoutReport,
342 // One refusal so metrics.config_refusals is exercised by the
343 // allowlist walk and not merely permitted by it.
344 ConfigRefusals: []status.ConfigRefusal{
345 {Host: "contract-refused.internal", Reason: "cannot_open", Since: time.Now().Add(-time.Hour), Last: time.Now(), Count: 2},
346 },
347 ConfigRefusalsTotal: 1,
348 // One rolled-back build, so metrics.release_refusals is exercised
349 // by the allowlist walk too.
350 ReleaseRefusals: []status.ReleaseRefusal{
351 {Host: "contract-rolled-back.internal", Version: "v2", Reason: "exited: status=2", RefusedAt: time.Now(),
352 Kind: "wrong-program", RecoveryError: "rollback did not come up either"},
353 },
354 // One host held for a dev proof, so metrics.proof_waits is exercised
355 // by the allowlist walk too.
356 ProofWaits: []status.ProofWait{
357 {Host: "contract-proof-wait.internal", Version: "v3", SHA256: "e47fcc1e0123456789abcdef", Prover: "dev",
358 Reason: "no proof from dev", Since: time.Now().Add(-time.Hour)},
359 },
360 // One host's build-state builds, so metrics.build_states is exercised
361 // by the allowlist walk too.
362 BuildStates: map[string]status.BuildStatePins{
363 "contract-held.internal": {Account: "bab", App: "km", Builds: []string{"v1", "v2"}, Complete: true},
364 },
365 // One host's update decision, so metrics.updates is exercised by the
366 // allowlist walk too.
367 UpdateSettings: status.UpdateSettingsProtocol,
368 Updates: map[string]status.HostUpdates{
369 "contract-decided.internal": {Desired: "v2", Reason: "major", Detail: "v2", Baking: "v3", Major: &contractMajor, Pending: 4, Completed: 3, CompletedFailed: true, Account: "bab", App: "km"},
370 },
371 ConfigGuard: "bypassed (CARLOS_NO_SEALED_CONFIG)",
372 MemFrac: &memFrac,
373 DiskFrac: &diskFrac,
374 Listening: listening,
375 LagSecs: &lagSecs,
376 AdoptRunning: &adopting,
377 MemTotalBytes: &memTotal,
378 MemAvailBytes: &memAvail,
379 Load1: &load1,
380 DiskFreeBytes: &diskFree,
381 Release: "contract-fixture-sha", // distinct from dryrun.go's "dryrun4242sha"
382 Tunnel: "connected", // distinct from dryrun.go's "reconnecting"
383 }
384}
385
386// TestBoxSchemaAllowlisted verifies that Box's JSON output carries only
387// fields listed in allowedBoxKeys. Any new field added to Box must be
388// explicitly added to allowedBoxKeys to pass this test — that's the guard.
389func TestBoxSchemaAllowlisted(t *testing.T) {
390 b := status.CollectBox(fixtureRoutes(), fixtureProbes(), time.Now())
391 j, err := json.Marshal(b)
392 if err != nil {
393 t.Fatalf("marshal Box: %v", err)
394 }
395 assertOnlyKeys(t, j, allowedBoxKeys)
396}
397
398// TestFleetSchemaAllowlisted verifies that Fleet's JSON output carries only
399// fields listed in allowedFleetKeys. Same guard as TestBoxSchemaAllowlisted,
400// but for the merged feed: new fields must be explicitly allowlisted.
401func TestFleetSchemaAllowlisted(t *testing.T) {
402 s := teststore(t)
403 now := time.Now()
404
405 // Write a box so BuildFleet has something to merge.
406 if err := status.WriteBox(context.Background(), s,
407 status.CollectBox(fixtureRoutes(), fixtureProbes(), now)); err != nil {
408 t.Fatalf("WriteBox: %v", err)
409 }
410
411 // Build the fleet over the teststore with routes that produce apps/services.
412 f, err := status.BuildFleet(context.Background(), s, fixtureRoutes(), now, 5*time.Minute)
413 if err != nil {
414 t.Fatalf("BuildFleet: %v", err)
415 }
416
417 j, err := json.Marshal(f)
418 if err != nil {
419 t.Fatalf("marshal Fleet: %v", err)
420 }
421 assertOnlyKeys(t, j, allowedFleetKeys)
422}
423
424// TestBuildFleetIgnoresFleetPrefixedSnapshots pins the customer-fleets
425// plan's Task 5 Step 3 contract: BuildFleet's loadBoxes lists EXACTLY
426// "status/boxes/" (fleet.go:197) — a fleet box's heartbeat, written under
427// "fleets/<name>/status/boxes/<label>.json" per the fleet box env contract
428// (CARLOS_STATUS_PREFIX, internal/host's newPrefixStore), sits outside that
429// prefix entirely and so never surfaces on the platform's own public
430// fleet.json. This is owner-only visibility BY CONSTRUCTION — there is no
431// allowlist rule to bypass, loadBoxes simply never reads that path — the
432// same shape FleetBox never copying Box's "metrics" key already relies on.
433func TestBuildFleetIgnoresFleetPrefixedSnapshots(t *testing.T) {
434 s := teststore(t)
435 now := time.Now()
436
437 // A fleet box's own snapshot, parked at the exact key its
438 // CARLOS_STATUS_PREFIX=fleets/pi/ produces — same Box shape a platform
439 // box's WriteBox would produce, just outside status/boxes/.
440 fleetBox := boxFixture("pi-box", "plenty", now)
441 body, err := json.Marshal(fleetBox)
442 if err != nil {
443 t.Fatal(err)
444 }
445 if _, err := s.Put(context.Background(), "fleets/pi/status/boxes/pi-box.json", body, ""); err != nil {
446 t.Fatal(err)
447 }
448
449 f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute)
450 if err != nil {
451 t.Fatalf("BuildFleet: %v", err)
452 }
453 if len(f.Boxes) != 0 {
454 t.Errorf("Boxes = %+v, want none — a fleet-prefixed snapshot must never surface on the public fleet page", f.Boxes)
455 }
456
457 // Contrast case: a platform box's own snapshot, written through the
458 // real WriteBox (the unprefixed key), MUST surface — proving the
459 // exclusion above is specifically the fleets/ prefix, not a broken
460 // loadBoxes that would hide everything.
461 if err := status.WriteBox(context.Background(), s, boxFixture("platform-box", "plenty", now)); err != nil {
462 t.Fatal(err)
463 }
464 f2, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute)
465 if err != nil {
466 t.Fatalf("BuildFleet: %v", err)
467 }
468 if len(f2.Boxes) != 1 || f2.Boxes[0].Label != "platform-box" {
469 t.Errorf("Boxes = %+v, want exactly platform-box (the fleet-prefixed snapshot still must not appear)", f2.Boxes)
470 }
471}
472
473// TestNoDigitsInBandSubtrees verifies that the band words (load, capacity)
474// never carry numeric digits. The privacy commitment: never publish raw
475// fractions or counts, only words. A digit in these fields would violate
476// that promise — and could leak enough information for an observer to
477// reconstruct the missing band threshold.
478func TestNoDigitsInBandSubtrees(t *testing.T) {
479 b := status.CollectBox(fixtureRoutes(), fixtureProbes(), time.Now())
480 for name, v := range map[string]*string{"load": b.Load, "capacity": b.Capacity} {
481 if v != nil && strings.ContainsAny(*v, "0123456789") {
482 t.Errorf("%s carries a digit: %q", name, *v)
483 }
484 }
485}
486