amadan
host: arm the status-privacy tripwire, and keep a loop panic off the box
The status-publish privacy test iterated store.List("status/") and
passed vacuously if it came back empty — a tripwire across a doorway
nobody walks through. It now fails if the publish tick wrote no status
keys at all.
metricsLoop also recovers at the top of its goroutine: it is always on
and shares a process with the agent serving this box's traffic, so a
panic in it must stop monitoring, not the box.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2 files changed,
+16
−0
internal/host/host_test.go+6 −0internal/host/metrics_loop.go+10 −0
diff --git a/internal/host/host_test.go b/internal/host/host_test.go| index d5171d04e..22033ae5d 100644 |
| --- a/internal/host/host_test.go |
| +++ b/internal/host/host_test.go |
| @@ -532,6 +532,12 @@ func TestStatusPublishNeverReadsMetricsPrefixes(t *testing.T) { |
| if err != nil { |
| t.Fatal(err) |
| } |
| + // Without this the loop below has nothing to inspect and the test passes |
| + // no matter what statusTick publishes — a tripwire across a doorway |
| + // nobody walks through. |
| + if len(keys) == 0 { |
| + t.Fatal("no status/ keys after a publish tick — the tripwire is not armed") |
| + } |
| for _, k := range keys { |
| body, _, err := store.Get(ctx, k) |
| if err != nil { |
diff --git a/internal/host/metrics_loop.go b/internal/host/metrics_loop.go| index f9df3a1a7..a0de7bd66 100644 |
| --- a/internal/host/metrics_loop.go |
| +++ b/internal/host/metrics_loop.go |
| @@ -700,6 +700,16 @@ func loadAlertState(path, box string) metrics.AlertState { |
| // alertTick runs every tick, since a threshold breach deserves faster |
| // detection than a metrics flush does. |
| func metricsLoop() { |
| + // This loop is always on and shares a process with the agent that serves |
| + // this box's traffic, so a panic in it — an unreadable cgroup shape, a |
| + // nil no one expected — must not take the box down with it. Monitoring |
| + // stopping is a bad day; the box stopping is an outage. |
| + defer func() { |
| + if r := recover(); r != nil { |
| + log.Printf("agent: metrics loop panic, monitoring stopped for this process: %v", r) |
| + } |
| + }() |
| + |
| ctx := context.Background() |
| bucket := resolveStatusBucket(os.Getenv) |
| if bucket == "" { |