amadan
status: publish config refusals in the private Metrics block (#213)
Box.Metrics.ConfigRefusals lists the hosts a box is refusing and why, as closed reason words, plus a total when capped. Private like every hostname-bearing field; the allowlist contract and the dry-run leak scan cover it.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
4 files changed,
+150
−16
internal/status/contract_test.go+25 −15internal/status/dryrun.go+31 −0internal/status/snapshot.go+42 −1internal/status/status_test.go+52 −0
diff --git a/internal/status/contract_test.go b/internal/status/contract_test.go| index 2aac4256b..bd6866f7d 100644 |
| --- a/internal/status/contract_test.go |
| +++ b/internal/status/contract_test.go |
| @@ -75,10 +75,14 @@ var allowedBoxKeys = map[string][]string{ |
| "": {"generated_at", "label", "region", "routes", "load", "capacity", "replication", "adoption_running", "config_path_running", "restart_path_running", "expired_canaries", "metrics"}, |
| "routes": {"static", "resident", "hibernating", "awake", "asleep"}, |
| "replication": {"healthy", "lag_seconds"}, |
| - "metrics": {"mem_total_bytes", "mem_avail_bytes", "load1", "disk_free_bytes", "release", "tunnel", "route_versions", "upstreams"}, |
| + "metrics": {"mem_total_bytes", "mem_avail_bytes", "load1", "disk_free_bytes", "release", "tunnel", "route_versions", "upstreams", "config_refusals", "config_refusals_total"}, |
| "metrics.route_versions": {wildcardAnyKey}, |
| "metrics.upstreams": {wildcardAnyKey}, |
| "metrics.upstreams.*[]": {"addr", "up", "since", "fails", "last_error"}, |
| + // config_refusals (amadan #213) names hosts and a closed reason word, in |
| + // the private block only. Pinned to the field level so a future field on |
| + // the refusal is a publication decision too. |
| + "metrics.config_refusals[]": {"host", "reason", "since", "last", "count"}, |
| } |
| // allowedFleetKeys are THE schema for Fleet's JSON output. Same friction as |
| @@ -289,20 +293,26 @@ func fixtureProbes() status.Probes { |
| } |
| return status.Probes{ |
| - MachineID: "test-machine-id", |
| - Region: "us-west-1", |
| - Upstreams: upstreams, |
| - MemFrac: &memFrac, |
| - DiskFrac: &diskFrac, |
| - Listening: listening, |
| - LagSecs: &lagSecs, |
| - AdoptRunning: &adopting, |
| - MemTotalBytes: &memTotal, |
| - MemAvailBytes: &memAvail, |
| - Load1: &load1, |
| - DiskFreeBytes: &diskFree, |
| - Release: "contract-fixture-sha", // distinct from dryrun.go's "dryrun4242sha" |
| - Tunnel: "connected", // distinct from dryrun.go's "reconnecting" |
| + MachineID: "test-machine-id", |
| + Region: "us-west-1", |
| + Upstreams: upstreams, |
| + // One refusal so metrics.config_refusals is exercised by the |
| + // allowlist walk and not merely permitted by it. |
| + ConfigRefusals: []status.ConfigRefusal{ |
| + {Host: "contract-refused.internal", Reason: "cannot_open", Since: time.Now().Add(-time.Hour), Last: time.Now(), Count: 2}, |
| + }, |
| + ConfigRefusalsTotal: 1, |
| + MemFrac: &memFrac, |
| + DiskFrac: &diskFrac, |
| + Listening: listening, |
| + LagSecs: &lagSecs, |
| + AdoptRunning: &adopting, |
| + MemTotalBytes: &memTotal, |
| + MemAvailBytes: &memAvail, |
| + Load1: &load1, |
| + DiskFreeBytes: &diskFree, |
| + Release: "contract-fixture-sha", // distinct from dryrun.go's "dryrun4242sha" |
| + Tunnel: "connected", // distinct from dryrun.go's "reconnecting" |
| } |
| } |
diff --git a/internal/status/dryrun.go b/internal/status/dryrun.go| index bf1f99eea..25e02899f 100644 |
| --- a/internal/status/dryrun.go |
| +++ b/internal/status/dryrun.go |
| @@ -108,6 +108,14 @@ const ( |
| var dryRunUpstreamNeedles = []string{dryRunUpstreamHost, dryRunUpstreamAddr} |
| +// dryRunConfigRefusalHost is the config guard's needle (amadan #213): |
| +// Box.Metrics.ConfigRefusals names the hosts a keyless box refuses, so the |
| +// same scope split applies. REQUIRED in status/boxes/*.json and BANNED in |
| +// every other published object. |
| +const dryRunConfigRefusalHost = "refused-secret.internal" |
| + |
| +var dryRunConfigRefusalNeedles = []string{dryRunConfigRefusalHost} |
| + |
| // ScanForLeaks reports which of banned's strings appear anywhere in j. It is |
| // the single implementation both DryRun's self-check and |
| // contract_test.go's scanForLeaks test helper run — the test package can't |
| @@ -189,6 +197,12 @@ func dryRunFixtures() ([]registry.Route, Probes) { |
| DiskFreeBytes: &diskFree, |
| Release: "dryrun4242sha", |
| Tunnel: "reconnecting", |
| + // One refused host (amadan #213), so the scope check below has |
| + // something to require in the private block and ban elsewhere. |
| + ConfigRefusals: []ConfigRefusal{ |
| + {Host: dryRunConfigRefusalHost, Reason: "no_key", Since: time.Now().Add(-time.Hour), Last: time.Now(), Count: 2}, |
| + }, |
| + ConfigRefusalsTotal: 1, |
| // The probe table, keyed by a route host and carrying a backend |
| // address — see dryRunUpstreamNeedles. One up and one down upstream, |
| // so the down one's reason string is exercised too. |
| @@ -341,6 +355,9 @@ func DryRun(dir string, out io.Writer) error { |
| // upstreamKeySeen is the same positive half for the probe table |
| // (dryRunUpstreamNeedles) — Box.Metrics.Upstreams, Task 6. |
| upstreamKeySeen := false |
| + // configRefusalKeySeen is the same positive half for the config guard's |
| + // refusal list (dryRunConfigRefusalNeedles), amadan #213. |
| + configRefusalKeySeen := false |
| for _, key := range keys { |
| body, _, err := s.Get(ctx, key) |
| @@ -400,6 +417,17 @@ func DryRun(dir string, out io.Writer) error { |
| return fmt.Errorf("dry run: %s leaked an upstreams fixture %v — Upstreams must never reach the public feed", key, leaks) |
| } |
| + // dryRunConfigRefusalNeedles: the fourth scope split. The refusal list |
| + // names real hosts, so status/boxes/*.json is its one private home. |
| + if strings.HasPrefix(key, "status/boxes/") { |
| + if leaks := ScanForLeaks(body, dryRunConfigRefusalNeedles); len(leaks) == 0 { |
| + return fmt.Errorf("dry run: %s: expected config_refusals fixtures %v, found none — ConfigRefusals not attached?", key, dryRunConfigRefusalNeedles) |
| + } |
| + configRefusalKeySeen = true |
| + } else if leaks := ScanForLeaks(body, dryRunConfigRefusalNeedles); len(leaks) > 0 { |
| + return fmt.Errorf("dry run: %s leaked a config_refusals fixture %v — ConfigRefusals must never reach the public feed", key, leaks) |
| + } |
| + |
| // Page assets (index.html, status.css) aren't JSON and simply fail |
| // to decode here — silently skipped, since the band-digit check |
| // only means anything for objects that actually carry Load/Capacity |
| @@ -424,6 +452,9 @@ func DryRun(dir string, out io.Writer) error { |
| if !upstreamKeySeen { |
| return fmt.Errorf("dry run: no status/boxes/*.json object was scanned — upstreams scope check never ran") |
| } |
| + if !configRefusalKeySeen { |
| + return fmt.Errorf("dry run: no status/boxes/*.json object was scanned — config_refusals scope check never ran") |
| + } |
| fmt.Fprintln(out, "dry-run: no leaks") |
| return nil |
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go| index dd757d7dd..e75efb795 100644 |
| --- a/internal/status/snapshot.go |
| +++ b/internal/status/snapshot.go |
| @@ -130,6 +130,34 @@ type Probes struct { |
| // answer (no file yet, stat failed), means unknown and the host is left |
| // out of Metrics.RouteDBBytes rather than published as 0. |
| DBBytes func(dbPath string) *int64 |
| + |
| + // ConfigRefusals is the instances this box is CURRENTLY refusing to take |
| + // because it cannot provide their config (amadan #213), as the host's |
| + // config guard reports them; nil when it refuses nothing. They land in |
| + // the private Metrics block only: they name hosts. ConfigRefusalsTotal is |
| + // how many are refused in all when the guard capped the list. |
| + ConfigRefusals []ConfigRefusal |
| + ConfigRefusalsTotal int |
| +} |
| + |
| +// ConfigRefusal is one host this box is refusing to take because it cannot |
| +// provide that instance's config: no CARLOS_SECRETS_KEY, one that does not |
| +// parse, one that does not open the sealed config, or a store read it could |
| +// not complete (a box that cannot tell refuses). Reason is a closed word |
| +// (no_key, key_unparseable, cannot_open, store_error), never error text, so |
| +// nothing from the store or the crypto reaches a snapshot. Since is when the |
| +// host was first refused in this stretch, Last the most recent refusal, and |
| +// Count how many wake or admission attempts were refused. |
| +// |
| +// PRIVATE, like everything under Box.Metrics: the host is a real route name. |
| +// It is declared here, with the other wire types of the snapshot, so the |
| +// readers (console, operators) need not import the host agent to decode it. |
| +type ConfigRefusal struct { |
| + Host string `json:"host"` |
| + Reason string `json:"reason"` |
| + Since time.Time `json:"since"` |
| + Last time.Time `json:"last"` |
| + Count int `json:"count"` |
| } |
| // UpstreamState is one upstream's health as the edge's prober currently sees |
| @@ -248,6 +276,16 @@ type BoxMetrics struct { |
| // readers. A host whose file could not be sized is absent, never 0; the |
| // whole map is omitted when no route could be sized. |
| RouteDBBytes map[string]int64 `json:"route_db_bytes,omitempty"` |
| + // ConfigRefusals is the hosts this box is refusing to take because it |
| + // cannot provide their config (amadan #213). It is how an operator sees |
| + // that a box is keyless, or holds the wrong key, without reading its |
| + // journal: a keyed sibling takes these instances, and this list says why |
| + // this box did not. Hostname-bearing, so it lives in this private block |
| + // and nowhere else (BuildFleet never names Metrics). Omitted when empty. |
| + ConfigRefusals []ConfigRefusal `json:"config_refusals,omitempty"` |
| + // ConfigRefusalsTotal is the count of refused hosts when ConfigRefusals |
| + // was capped, so a long list reads as long. Omitted when zero. |
| + ConfigRefusalsTotal int `json:"config_refusals_total,omitempty"` |
| } |
| // AliasReading is one alias hostname's status as CollectBox saw it this |
| @@ -616,7 +654,7 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box { |
| p.DiskFreeBytes != nil || p.Release != "" || p.Tunnel != "" || |
| p.InstanceID != "" || p.Platform != "" || p.SystemChannel != "" || p.System != nil || |
| len(routeVersions) > 0 || len(aliasReadings) > 0 || len(p.Upstreams) > 0 || |
| - len(routeDBBytes) > 0 { |
| + len(routeDBBytes) > 0 || len(p.ConfigRefusals) > 0 { |
| box.Metrics = &BoxMetrics{ |
| MemTotalBytes: p.MemTotalBytes, MemAvailBytes: p.MemAvailBytes, |
| Load1: p.Load1, DiskFreeBytes: p.DiskFreeBytes, |
| @@ -632,6 +670,9 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box { |
| // alongside another known field above, and omitempty drops it. |
| Upstreams: p.Upstreams, |
| RouteDBBytes: routeDBBytes, |
| + // A fresh copy: the guard's own slice is its to mutate. |
| + ConfigRefusals: append([]ConfigRefusal(nil), p.ConfigRefusals...), |
| + ConfigRefusalsTotal: p.ConfigRefusalsTotal, |
| } |
| } |
diff --git a/internal/status/status_test.go b/internal/status/status_test.go| index 4c6f12fbe..8d6c2d195 100644 |
| --- a/internal/status/status_test.go |
| +++ b/internal/status/status_test.go |
| @@ -1444,3 +1444,55 @@ func TestCollectBoxCarriesRouteDBBytes(t *testing.T) { |
| t.Errorf("metrics = %+v with no DBBytes probe, want absent", b.Metrics) |
| } |
| } |
| + |
| +// TestCollectBoxCarriesConfigRefusals (amadan #213): a box that refuses to |
| +// take an instance because it cannot provide that instance's config says so |
| +// in its private Metrics block, where an operator can see which hosts and |
| +// why. The hostnames never reach the public fleet.json, and a box with no |
| +// refusal carries no key at all. |
| +func TestCollectBoxCarriesConfigRefusals(t *testing.T) { |
| + since := time.Date(2026, 10, 5, 9, 0, 0, 0, time.UTC) |
| + refusals := []status.ConfigRefusal{ |
| + {Host: "refused-secret.example.com", Reason: "no_key", Since: since, Last: since.Add(time.Minute), Count: 3}, |
| + } |
| + routes := []registry.Route{{Host: "refused-secret.example.com", Kind: "instance", Hibernate: true}} |
| + b := status.CollectBox(routes, status.Probes{MachineID: "m", ConfigRefusals: refusals, ConfigRefusalsTotal: 5}, time.Now()) |
| + if b.Metrics == nil { |
| + t.Fatal("metrics block missing: a refusal alone should attach it") |
| + } |
| + if got := b.Metrics.ConfigRefusals; len(got) != 1 || got[0] != refusals[0] { |
| + t.Fatalf("config_refusals = %+v, want %+v", got, refusals) |
| + } |
| + if b.Metrics.ConfigRefusalsTotal != 5 { |
| + t.Errorf("config_refusals_total = %d, want 5", b.Metrics.ConfigRefusalsTotal) |
| + } |
| + |
| + s := teststore(t) |
| + ctx := context.Background() |
| + if err := status.WriteBox(ctx, s, b); err != nil { |
| + t.Fatal(err) |
| + } |
| + f, err := status.BuildFleet(ctx, s, routes, time.Now(), 5*time.Minute) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + fj, err := json.Marshal(f) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + for _, needle := range []string{"refused-secret.example.com", "no_key", "config_refusals"} { |
| + if strings.Contains(string(fj), needle) { |
| + t.Errorf("fleet.json leaked %q: %s", needle, fj) |
| + } |
| + } |
| + |
| + // Nothing refused: no Metrics block from this alone, no key on the wire. |
| + quiet := status.CollectBox(routes, status.Probes{MachineID: "m"}, time.Now()) |
| + if quiet.Metrics != nil { |
| + t.Errorf("metrics = %+v with no refusal, want absent", quiet.Metrics) |
| + } |
| + raw, _ := json.Marshal(status.CollectBox(routes, status.Probes{MachineID: "m", MemTotalBytes: new(int64)}, time.Now())) |
| + if strings.Contains(string(raw), "config_refusals") { |
| + t.Errorf("a box with no refusal publishes config_refusals: %s", raw) |
| + } |
| +} |