amadan
status: an unarmed restart trigger is visible on the box row
carlos-restart.path gets the probe carlos-adopt.path already had, and travels the same chain: Probes.RestartPathRunning -> Box -> FleetBox as restart_path_running. Same silent failure mode (enable writes a symlink, only start arms the watch), sharper consequence — unarmed, a member is told 'requested' and waits out the hourly pass for a cycle they were promised in seconds, and nothing inside the agent can tell: it writes its marker either way. Wire-additive; pre-#113 snapshots decode to null.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
6 files changed,
+125
−28
internal/host/probes.go+19 −0internal/status/contract_test.go+9 −2internal/status/dryrun.go+10 −8internal/status/fleet.go+14 −8internal/status/snapshot.go+26 −10internal/status/status_test.go+47 −0
diff --git a/internal/host/probes.go b/internal/host/probes.go| index 6a64ce1c4..5264137f3 100644 |
| --- a/internal/host/probes.go |
| +++ b/internal/host/probes.go |
| @@ -62,6 +62,25 @@ func configPathActive() *bool { |
| return parseIsActive(string(out)) |
| } |
| +// restartPathUnit is the unit whose liveness IS "a member's `carlos restart` |
| +// reaches this box's units in seconds" (issue #113): the path unit watching |
| +// /data/restart-requested, which the agent touches and which starts |
| +// carlos-restart.service. Installed and armed by bootstrap alongside the other |
| +// two (see enabledUnits). |
| +const restartPathUnit = "carlos-restart.path" |
| + |
| +// restartPathActive asks the same question configPathActive asks, about a unit |
| +// with the same silent failure mode — and one worse consequence. An unarmed |
| +// config trigger costs an env var an hour; an unarmed restart trigger means |
| +// someone typed `carlos restart` at a wedged instance, was told "requested", |
| +// and will stand there watching a process that will not cycle until the hourly |
| +// pass comes round. Nothing inside the agent can see the difference: it writes |
| +// its marker either way. |
| +func restartPathActive() *bool { |
| + out, _ := exec.Command("systemctl", "is-active", restartPathUnit).Output() |
| + return parseIsActive(string(out)) |
| +} |
| + |
| // parseIsActive maps systemd's is-active vocabulary onto the tri-state the |
| // status page publishes. "unknown" — systemd's answer when it has no such unit |
| // loaded — is a genuine false: a box with no adoption timer adopts nothing. |
diff --git a/internal/status/contract_test.go b/internal/status/contract_test.go| index 3c485445b..075569d76 100644 |
| --- a/internal/status/contract_test.go |
| +++ b/internal/status/contract_test.go |
| @@ -39,6 +39,13 @@ var allowedBoxKeys = map[string][]string{ |
| // on the hourly floor. Same publication argument — a platform-own unit's |
| // liveness on an opaquely-labelled box, no tenant in it anywhere. |
| // |
| + // "restart_path_running" is the issue #113 sibling of both: whether this |
| + // box's restart trigger unit (carlos-restart.path) is armed, i.e. whether a |
| + // member's `carlos restart` reaches its units in seconds or waits out the |
| + // hourly floor. Identical publication argument — one platform-own unit's |
| + // liveness on an opaquely-labelled box, and no tenant anywhere in it (it |
| + // says nothing about WHICH app was restarted, or that any was). |
| + // |
| // "metrics" is the one key in this allowlist that is EXEMPT from the |
| // band-word-not-a-number rule every other Box field follows (Task 5, |
| // internal/status/snapshot.go's package doc): it carries raw byte |
| @@ -56,7 +63,7 @@ var allowedBoxKeys = map[string][]string{ |
| // safe for exactly the same bucket-private reason: BuildFleet's |
| // field-by-field copy never names "metrics" at all, so no hostname here |
| // can reach FleetBox or fleet.json regardless of what this map holds. |
| - "": {"generated_at", "label", "region", "routes", "load", "capacity", "replication", "adoption_running", "config_path_running", "expired_canaries", "metrics"}, |
| + "": {"generated_at", "label", "region", "routes", "load", "capacity", "replication", "adoption_running", "config_path_running", "restart_path_running", "expired_canaries", "metrics"}, |
| "routes": {"static", "resident", "hibernating", "awake", "asleep"}, |
| "replication": {"healthy", "lag_seconds"}, |
| "metrics": {"mem_total_bytes", "mem_avail_bytes", "load1", "disk_free_bytes", "release", "tunnel", "route_versions"}, |
| @@ -67,7 +74,7 @@ var allowedBoxKeys = map[string][]string{ |
| // allowedBoxKeys: every new field is a deliberate publication decision. |
| var allowedFleetKeys = map[string][]string{ |
| "": {"generated_at", "boxes", "instances", "capacity", "apps", "apps_unlisted", "services", "replication", "adoption_running", "expired_canaries"}, |
| - "boxes[]": {"label", "region", "load", "stale", "routes", "adoption_running", "config_path_running", "expired_canaries"}, |
| + "boxes[]": {"label", "region", "load", "stale", "routes", "adoption_running", "config_path_running", "restart_path_running", "expired_canaries"}, |
| "boxes[].routes": {"static", "resident", "hibernating", "awake", "asleep"}, |
| "instances": {"total", "static", "resident", "hibernating", "awake", "asleep"}, |
| "capacity": {"level"}, |
diff --git a/internal/status/dryrun.go b/internal/status/dryrun.go| index 4143f75a0..aec3da27b 100644 |
| --- a/internal/status/dryrun.go |
| +++ b/internal/status/dryrun.go |
| @@ -141,6 +141,7 @@ func dryRunFixtures() ([]registry.Route, Probes) { |
| lag := int64(120) |
| adopting := true // a known reading, so adoption_running publishes non-null and gets scanned |
| configPath := true // ditto for config_path_running (issue #44) |
| + restartPath := true // ditto for restart_path_running (issue #113) |
| expiredCanaries := 2 // ditto for expired_canaries — a known count, not the unexercised nil path |
| // Task 5's raw Metrics fixtures: distinctive values (see |
| @@ -151,14 +152,15 @@ func dryRunFixtures() ([]registry.Route, Probes) { |
| load1 := 42.42 |
| probes := Probes{ |
| - MachineID: "ip-10-0-0-7", |
| - Region: "us-east-1", |
| - MemFrac: &memFrac, |
| - DiskFrac: &diskFrac, |
| - LagSecs: &lag, |
| - AdoptRunning: &adopting, |
| - ConfigPathRunning: &configPath, |
| - ExpiredCanaries: &expiredCanaries, |
| + MachineID: "ip-10-0-0-7", |
| + Region: "us-east-1", |
| + MemFrac: &memFrac, |
| + DiskFrac: &diskFrac, |
| + LagSecs: &lag, |
| + AdoptRunning: &adopting, |
| + ConfigPathRunning: &configPath, |
| + RestartPathRunning: &restartPath, |
| + ExpiredCanaries: &expiredCanaries, |
| // Every hibernating route reports "listening" so Awake/Asleep publish |
| // non-null — an unexercised null field is a field DryRun can't scan. |
| Listening: func(string) *bool { v := true; return &v }, |
diff --git a/internal/status/fleet.go b/internal/status/fleet.go| index eaf12a48a..463f4b656 100644 |
| --- a/internal/status/fleet.go |
| +++ b/internal/status/fleet.go |
| @@ -50,6 +50,11 @@ type FleetBox struct { |
| // page shows a tile for them, and this one is a per-box operational |
| // fact, not a headline about the fleet. |
| ConfigPathRunning *bool `json:"config_path_running"` |
| + // RestartPathRunning appears per-box for exactly the reason |
| + // ConfigPathRunning does, and is not rolled up for the same reason: it is a |
| + // per-box operational fact ("this box's restart trigger is not watching"), |
| + // not a headline about the fleet. |
| + RestartPathRunning *bool `json:"restart_path_running"` |
| // ExpiredCanaries appears per-box for the same reason AdoptionRunning |
| // does: the fleet-wide sum says something is expired somewhere, this |
| // says on which box, without naming which route. Published in the FEED |
| @@ -165,14 +170,15 @@ func BuildFleet(ctx context.Context, s release.Store, routes []registry.Route, n |
| for i, b := range boxes { |
| stale[i] = BoxStale(b, now, period) |
| f.Boxes = append(f.Boxes, FleetBox{ |
| - Label: b.Label, |
| - Region: b.Region, |
| - Load: b.Load, |
| - Stale: stale[i], |
| - Routes: b.Routes, |
| - AdoptionRunning: b.AdoptionRunning, |
| - ConfigPathRunning: b.ConfigPathRunning, |
| - ExpiredCanaries: b.ExpiredCanaries, |
| + Label: b.Label, |
| + Region: b.Region, |
| + Load: b.Load, |
| + Stale: stale[i], |
| + Routes: b.Routes, |
| + AdoptionRunning: b.AdoptionRunning, |
| + ConfigPathRunning: b.ConfigPathRunning, |
| + RestartPathRunning: b.RestartPathRunning, |
| + ExpiredCanaries: b.ExpiredCanaries, |
| }) |
| } |
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go| index 5fb6f128e..39b18fc08 100644 |
| --- a/internal/status/snapshot.go |
| +++ b/internal/status/snapshot.go |
| @@ -59,6 +59,13 @@ type Probes struct { |
| // half-working this field exists to end. nil is "never asked / couldn't |
| // tell" and publishes as null, same as every other probe here. |
| ConfigPathRunning *bool |
| + // RestartPathRunning is whether this box's restart trigger unit |
| + // (carlos-restart.path) is ACTIVE — ConfigPathRunning's issue #113 sibling, |
| + // same failure mode, sharper consequence: with it unarmed, a member who |
| + // runs `carlos restart` against a wedged instance is told "requested" and |
| + // waits out the hourly pass for a cycle they were led to expect in |
| + // seconds. nil is "never asked / couldn't tell" and publishes as null. |
| + RestartPathRunning *bool |
| // ExpiredCanaries is how many of this box's own canary/* routes have |
| // passed release.CanaryTTL (pipelines spec §5 "Canaries expire"), |
| // pre-computed by the caller against the release store — CollectBox |
| @@ -171,6 +178,14 @@ type Box struct { |
| // issue #44 carry no such key at all and decode to null, which is the |
| // honest reading of a box that never reported it. |
| ConfigPathRunning *bool `json:"config_path_running"` |
| + // RestartPathRunning is whether a member's `carlos restart` reaches this |
| + // box's units in seconds (carlos-restart.path armed) or only on the hourly |
| + // floor: true = armed, false = installed but not watching, null = unknown. |
| + // Same content and same privacy argument as ConfigPathRunning above — the |
| + // liveness of one of the platform's own units on an opaquely-labelled box. |
| + // Snapshots written before issue #113 carry no such key and decode to null, |
| + // which is the honest reading of a box that never reported it. |
| + RestartPathRunning *bool `json:"restart_path_running"` |
| // ExpiredCanaries publishes Probes.ExpiredCanaries straight through — a |
| // count, never hostnames (pipelines spec §5): which canary route expired |
| // is exactly the kind of identifying detail this package's whole design |
| @@ -353,16 +368,17 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box { |
| } |
| box := Box{ |
| - GeneratedAt: now, |
| - Label: Label(p.MachineID), |
| - Region: p.Region, |
| - Routes: counts, |
| - Load: bandWord(p.MemFrac, loadWords), |
| - Capacity: bandWord(worst, capacityWords), |
| - Replication: Replication{Healthy: healthy, LagSecs: p.LagSecs}, |
| - AdoptionRunning: p.AdoptRunning, |
| - ConfigPathRunning: p.ConfigPathRunning, |
| - ExpiredCanaries: p.ExpiredCanaries, |
| + GeneratedAt: now, |
| + Label: Label(p.MachineID), |
| + Region: p.Region, |
| + Routes: counts, |
| + Load: bandWord(p.MemFrac, loadWords), |
| + Capacity: bandWord(worst, capacityWords), |
| + Replication: Replication{Healthy: healthy, LagSecs: p.LagSecs}, |
| + AdoptionRunning: p.AdoptRunning, |
| + ConfigPathRunning: p.ConfigPathRunning, |
| + RestartPathRunning: p.RestartPathRunning, |
| + ExpiredCanaries: p.ExpiredCanaries, |
| } |
| // Metrics attaches only when something is known — an all-unknown Probes |
diff --git a/internal/status/status_test.go b/internal/status/status_test.go| index 928b9d609..4e83e905b 100644 |
| --- a/internal/status/status_test.go |
| +++ b/internal/status/status_test.go |
| @@ -758,6 +758,53 @@ func TestConfigPathRunningIsWireAdditive(t *testing.T) { |
| } |
| } |
| +// TestRestartPathRunningTravelsPerBox is TestConfigPathRunningTravelsPerBox's |
| +// issue #113 twin, and it exists for a sharper reason than symmetry: with |
| +// carlos-restart.path unarmed, a member runs `carlos restart` at a wedged |
| +// instance, is told "requested", and waits for a cycle that will not come until |
| +// the hourly pass. Nothing inside the agent can see that — it writes its marker |
| +// either way — so this reading is the only thing that can. Unknown stays null. |
| +func TestRestartPathRunningTravelsPerBox(t *testing.T) { |
| + if b := status.CollectBox(nil, status.Probes{MachineID: "m"}, time.Now()); b.RestartPathRunning != nil { |
| + t.Errorf("restart_path_running = %v, want null when the probe never answered", *b.RestartPathRunning) |
| + } |
| + for _, want := range []bool{true, false} { |
| + got := status.CollectBox(nil, status.Probes{MachineID: "m", RestartPathRunning: &want}, time.Now()).RestartPathRunning |
| + if got == nil || *got != want { |
| + t.Fatalf("RestartPathRunning=%v: box field = %v, want %v", want, got, want) |
| + } |
| + } |
| + |
| + s := teststore(t) |
| + now := time.Now() |
| + no := false |
| + b := status.CollectBox(nil, status.Probes{MachineID: "m", RestartPathRunning: &no}, now) |
| + if err := status.WriteBox(context.Background(), s, b); err != nil { |
| + t.Fatal(err) |
| + } |
| + f, err := status.BuildFleet(context.Background(), s, nil, now, 5*time.Minute) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(f.Boxes) != 1 { |
| + t.Fatalf("fleet boxes = %d, want 1", len(f.Boxes)) |
| + } |
| + if got := f.Boxes[0].RestartPathRunning; got == nil || *got { |
| + t.Fatalf("fleet box restart_path_running = %v, want false — knowing WHICH box cannot cycle a wedged instance is the whole value", got) |
| + } |
| + |
| + // Wire-additive, like every field before it: a snapshot written before |
| + // issue #113 has no such key and must read as unknown, not a healthy false. |
| + var old status.Box |
| + raw := `{"generated_at":"2026-08-04T00:00:00Z","label":"box-old","routes":{"static":0,"resident":0,"hibernating":0,"awake":null,"asleep":null},"load":null,"capacity":null,"replication":{"healthy":null,"lag_seconds":null},"adoption_running":true,"config_path_running":true,"expired_canaries":null}` |
| + if err := json.Unmarshal([]byte(raw), &old); err != nil { |
| + t.Fatalf("a pre-#113 snapshot must still parse: %v", err) |
| + } |
| + if old.RestartPathRunning != nil { |
| + t.Fatalf("restart_path_running = %v, want null for a snapshot written before the field existed", *old.RestartPathRunning) |
| + } |
| +} |
| + |
| // TestFleetAdoptionWorstOf covers the aggregate: a box that is KNOWN not to be |
| // adopting is a defect the fleet number must never hide, so one false outranks |
| // any number of trues; an unknown box can't be summed into a confident true, |