amadan

carlos / platform Public

host: the sampler scans the delegated exec subtree as a third root

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Paul Campbell e79a1f564a5d4d14b3df88e28463b7982a615105 parent 38f6c0a
4 files changed, +90 −8
  • internal/host/agent.go +1 −1
  • internal/host/metrics_loop.go +10 −7
  • internal/host/metrics_probes.go +35 −0
  • internal/host/metrics_probes_test.go +44 −0
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 53743493c..bc5429a14 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -161,7 +161,7 @@ func RunAgent(args []string) {
go statusLoop(reg)
go domainsLoop(reg, peekAcmeCache(args))
go instancesLoop(reg)
- go metricsLoop()
+ go metricsLoop("")
go configSyncLoop(reg, svc, dd)
go configGenLoop(reg, svc, dd)
go socketAdvisoryLoop(reg, svc)
diff --git a/internal/host/metrics_loop.go b/internal/host/metrics_loop.go
index a0de7bd66..4c7045cec 100644
--- a/internal/host/metrics_loop.go
+++ b/internal/host/metrics_loop.go
@@ -618,13 +618,13 @@ func httpPostWebhook(url string, body []byte) error {
// metrics_probes.go and probes.go — the production seam metricsLoop uses,
// never called from a test (host_test.go's convention: tests inject
// fixtures, never touch /proc or a syscall).
-func realSampleProbes() sampleProbes {
+func realSampleProbes(execRoot string) sampleProbes {
return sampleProbes{
procStat: readProcStat,
loadavg: readLoadAvg,
memFrac: readMemFrac,
diskFrac: readDiskFrac,
- units: realUnitReadings,
+ units: func() map[string]unitRaw { return realUnitReadings(execRoot) },
now: time.Now,
}
}
@@ -633,10 +633,13 @@ func realSampleProbes() sampleProbes {
// each one's cgroup usage — the production units func sampleProbes.units
// wraps. Keyed by UNIT NAME (what the console matches against), read from the
// unit's own cgroup DIRECTORY (which for a tenant instance is a level down;
-// see listCarlosUnits).
-func realUnitReadings() map[string]unitRaw {
+// see listCarlosUnits) — plus, when delegation is armed, the per-host
+// exec-child cgroups under execRoot (keyed carlos-exec@<host>; see
+// listExecCgroups).
+func realUnitReadings(execRoot string) map[string]unitRaw {
units := listCarlosUnits(cgroupSliceDir)
- if units == nil {
+ units = append(units, listExecCgroups(execRoot)...)
+ if len(units) == 0 {
return nil
}
out := make(map[string]unitRaw, len(units))
@@ -699,7 +702,7 @@ func loadAlertState(path, box string) metrics.AlertState {
// uses. Flush runs every 5th tick (5 minutes at the 60s tick cadence);
// alertTick runs every tick, since a threshold breach deserves faster
// detection than a metrics flush does.
-func metricsLoop() {
+func metricsLoop(execRoot string) {
// This loop is always on and shares a process with the agent that serves
// this box's traffic, so a panic in it — an unreadable cgroup shape, a
// nil no one expected — must not take the box down with it. Monitoring
@@ -732,7 +735,7 @@ func metricsLoop() {
log.Printf("agent: metrics loop armed (60s ticks, 5min flush), publishing to bucket %s", bucket)
sm := &sampler{}
- probes := realSampleProbes()
+ probes := realSampleProbes(execRoot)
tick := 0
for {
time.Sleep(60 * time.Second)
diff --git a/internal/host/metrics_probes.go b/internal/host/metrics_probes.go
index 899247438..8ad11bd6e 100644
--- a/internal/host/metrics_probes.go
+++ b/internal/host/metrics_probes.go
@@ -15,6 +15,7 @@ import (
"path/filepath"
"strings"
+ "github.com/carlosframework/platform/internal/cgroup"
"github.com/carlosframework/platform/internal/metrics"
)
@@ -164,3 +165,37 @@ func readUnitRaw(unitPath string) unitRaw {
return raw
}
+
+// listExecCgroups lists the per-host exec-child cgroups under the agent's
+// delegated subtree — the third scan root, alongside listCarlosUnits' two
+// levels. An empty root (no delegation armed) or a read failure -> nil. The
+// agent's own supervise leaf fails IsExecName and is skipped: that process
+// is already reported as carlos-edge.service. A dir whose cgroup.procs is
+// empty or unreadable is skipped too: a failed seed or an aborted wake
+// leaves the dir behind (until a later exit's rmdir or the next agent
+// start's sweep), and reporting it would render a host whose wake FAILED
+// as a live instance at zero — absence is the data, never a guessed zero.
+// readUnitRaw reads the included dirs unchanged — same cgroup v2 files,
+// same fail-toward-nil fields.
+func listExecCgroups(root string) []unitCgroup {
+ if root == "" {
+ return nil
+ }
+ entries, err := os.ReadDir(root)
+ if err != nil {
+ return nil
+ }
+ var units []unitCgroup
+ for _, entry := range entries {
+ if !entry.IsDir() || !cgroup.IsExecName(entry.Name()) {
+ continue
+ }
+ dir := filepath.Join(root, entry.Name())
+ procs, err := os.ReadFile(filepath.Join(dir, "cgroup.procs"))
+ if err != nil || len(strings.TrimSpace(string(procs))) == 0 {
+ continue
+ }
+ units = append(units, unitCgroup{name: entry.Name(), dir: dir})
+ }
+ return units
+}
diff --git a/internal/host/metrics_probes_test.go b/internal/host/metrics_probes_test.go
index 8ea66188d..0e691d0ca 100644
--- a/internal/host/metrics_probes_test.go
+++ b/internal/host/metrics_probes_test.go
@@ -88,3 +88,47 @@ func TestReadUnitRawReadsFromUnitDir(t *testing.T) {
t.Errorf("absent files must stay nil, got peak=%v r=%v w=%v", raw.memPeakBytes, raw.readBytes, raw.writeBytes)
}
}
+
+// TestListExecCgroupsFindsPerHostDirs pins the delegated subtree's scan:
+// carlos-exec@<host> dirs with live members come out with their names
+// VERBATIM as sample keys (deliberately not dressed up as .service units —
+// an operator who takes a dashboard row to systemctl must not be told a
+// unit exists that doesn't). The agent's own supervise leaf and foreign
+// names are invisible; so is a MEMBER-LESS dir (empty or missing
+// cgroup.procs) — a failed seed's leftover must read as absent, never as a
+// live instance at zero. An empty root — the no-delegation state — scans
+// nothing.
+func TestListExecCgroupsFindsPerHostDirs(t *testing.T) {
+ root := t.TempDir()
+ mk := func(name, procs string) string {
+ t.Helper()
+ dir := filepath.Join(root, name)
+ if err := os.MkdirAll(dir, 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if procs != "" {
+ if err := os.WriteFile(filepath.Join(dir, "cgroup.procs"), []byte(procs), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ }
+ return dir
+ }
+ execDir := mk("carlos-exec@keymail.dev", "1234\n")
+ mk("supervise", "1\n")
+ mk("carlos-exec@", "99\n") // no host — not one of ours
+ mk("random-dir", "77\n") // foreign name
+ mk("carlos-exec@failed.example.com", "\n") // member-less: whitespace-only procs
+ mk("carlos-exec@aborted.example.com", "") // member-less: no procs file at all
+
+ got := listExecCgroups(root)
+ if len(got) != 1 {
+ t.Fatalf("discovered %d cgroups, want 1: %+v", len(got), got)
+ }
+ if got[0].name != "carlos-exec@keymail.dev" || got[0].dir != execDir {
+ t.Fatalf("got %+v, want name carlos-exec@keymail.dev dir %s", got[0], execDir)
+ }
+
+ if out := listExecCgroups(""); out != nil {
+ t.Fatalf("empty root must scan nothing, got %+v", out)
+ }
+}