amadan

carlos / platform Public

status/console/cli: render raw host metrics privately, stale hosts as unreachable

Task 5 gave Box a raw Metrics block and Task 6 filled it; nothing read it
back. This surfaces it on the two surfaces allowed to see raw numbers — the
session-gated operator fleet page and an operator's own shell — and nowhere
else: fleet.json's shape and the public status page are untouched.

status gains LoadBoxes (the exported loadBoxes: one parser of the
status/boxes/ shape, one skip-corrupt rule) and BoxStale, which BuildFleet's
own per-box staleness now calls, so the published boxes table and every
private reader can never drift into disagreeing about which host is dark.
SnapshotPeriod moves the 5-minute tick into the status package, since the
console and the CLI have no agent-local const to borrow.

The rule both renderings enforce: an unreachable host shows how long it has
been dark and NOT ONE of its last readings. On the console that suppression
happens in the view model (HostRow.Metrics is nil'd for a stale box), not in
the template, so a dead box's numbers are never handed to the page at all.
The host detail is an extra read on a page that already had its answer, so a
boxes read that fails costs the section and a log line, never the page.

carlos status grows -live, reading the same bucket the agent's status loop
writes (host.ResolveStatusBucket, exported over the existing helper).
-dry-run is byte-identical; the usage text's "no live mode" paragraph is now
false and gone.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Paul Campbell 4137c3ef98cb51a584dc52cc9130e11b7f0f6df0 parent d3df023
9 files changed, +655 −17
  • cli.go +119 −11
  • internal/console/server.go +28 −1
  • internal/console/server_test.go +127 −0
  • internal/console/templates/fleet.html +36 −0
  • internal/console/view.go +84 −0
  • internal/host/agent.go +14 −2
  • internal/status/fleet.go +46 −3
  • internal/status/status_test.go +74 −0
  • statuslive_test.go +127 −0
diff --git a/cli.go b/cli.go
index a3f268290..b4a56b284 100644
--- a/cli.go
+++ b/cli.go
@@ -19,12 +19,14 @@ import (
"os"
"path/filepath"
"regexp"
+ "sort"
"strings"
"text/tabwriter"
"time"
"github.com/carlosframework/platform/internal/appconfig"
"github.com/carlosframework/platform/internal/edge"
+ "github.com/carlosframework/platform/internal/host"
"github.com/carlosframework/platform/internal/ledger"
"github.com/carlosframework/platform/internal/registry"
"github.com/carlosframework/platform/internal/release"
@@ -1643,12 +1645,13 @@ func runKeygen(args []string) {
}
// statusUsage is shared between "no flag given" and "-h/--help" so an
-// operator gets the same explanation either way: this verb only has a
-// dry-run mode. The fleet status page's LIVE publish loop (CollectBox ->
-// WriteBox on every box, BuildFleet -> PublishFleet on the lease holder)
-// runs inside the box agent's own 5-minute tick, not from a one-shot CLI
-// invocation — there is deliberately no "carlos status publish" here.
+// operator gets the same explanation either way: this verb reads and checks,
+// it never publishes. The fleet status page's PUBLISH loop (CollectBox ->
+// WriteBox on every box, BuildFleet -> PublishFleet on the lease holder) runs
+// inside the box agent's own 5-minute tick — there is deliberately no
+// "carlos status publish" here; -live only reads what those ticks wrote.
const statusUsage = `usage: carlos status -dry-run <dir>
+ carlos status -live
-dry-run <dir> build a throwaway fleet stuffed with fixture secrets
(hostnames, a machine id, mid-band fractions), run it
@@ -1659,19 +1662,32 @@ const statusUsage = `usage: carlos status -dry-run <dir>
scratch directory before ever pointing this binary at a
real bucket.
-carlos status has no live mode: the fleet status page is published every 5
-minutes by the box agent's own status lease holder (see "carlos agent"),
-never by a one-shot CLI invocation.
+ -live read status/boxes/*.json out of the status bucket and print
+ one line per host: release, tunnel state, memory, load and
+ free disk — raw figures, never published on the public
+ status page. A host that has stopped reporting prints as
+ UNREACHABLE with how long it has been dark and none of its
+ last readings. Needs bucket credentials and
+ CARLOS_STATUS_BUCKET (or CARLOS_DEPLOYMENT_BUCKET, the same
+ defaulting the agent's own status loop applies).
`
-// RunStatus is the "carlos status" verb: dry-run only. Modeled on ops.Run's
-// FlagSet shape (main.go's "ops" case), but flatter — status has exactly one
-// mode today, so there is no sub-verb dispatch to build out yet.
+// RunStatus is the "carlos status" verb: a fixture dry-run and a live read of
+// what the fleet's boxes last reported. Modeled on ops.Run's FlagSet shape
+// (main.go's "ops" case), but flatter — two independent modes, no sub-verb
+// dispatch.
func RunStatus(args []string) {
fs := flag.NewFlagSet("status", flag.ExitOnError)
dryRunDir := fs.String("dry-run", "", "build a fixture fleet, publish it into this directory, and verify no leaks")
+ liveFlag := fs.Bool("live", false, "read status/boxes/*.json from the status bucket and print per-host metrics")
fs.Parse(args)
+ if *liveFlag {
+ if err := statusLive(os.Stdout); err != nil {
+ log.Fatal(err)
+ }
+ return
+ }
if *dryRunDir == "" {
fmt.Fprint(os.Stderr, statusUsage)
os.Exit(1)
@@ -1680,3 +1696,95 @@ func RunStatus(args []string) {
log.Fatal(err)
}
}
+
+// statusLive prints what every box in the fleet last reported about itself,
+// read straight from status/boxes/ — the private snapshots, raw numbers and
+// all, which is why this needs the operator's own bucket credentials and why
+// nothing here is ever served publicly (see internal/status's package
+// comment). Bucket resolution is host.ResolveStatusBucket, the identical
+// CARLOS_STATUS_BUCKET-then-CARLOS_DEPLOYMENT_BUCKET rule the agent's status
+// loop uses, so this reads exactly the bucket the agents write.
+func statusLive(w io.Writer) error {
+ ctx := context.Background()
+ bucket := host.ResolveStatusBucket(os.Getenv)
+ if bucket == "" {
+ return fmt.Errorf("set CARLOS_STATUS_BUCKET or CARLOS_DEPLOYMENT_BUCKET")
+ }
+ s, err := release.NewS3Store(ctx, bucket)
+ if err != nil {
+ return err
+ }
+ boxes, err := status.LoadBoxes(ctx, s)
+ if err != nil {
+ return err
+ }
+ printLiveBoxes(w, boxes, time.Now())
+ return nil
+}
+
+// printLiveBoxes is statusLive's rendering half, split out so it can be
+// asserted on without a bucket. One line per box, label-sorted so two runs
+// diff on the fleet rather than on list order.
+//
+// A stale box (status.BoxStale — the same rule the console and the published
+// boxes table use) prints UNREACHABLE and how long it has been dark, and
+// deliberately NOT its last readings: those numbers described a machine that
+// has since stopped answering, and printing them in the same columns as a
+// live host's is how an outage gets read as fine. A fresh box that carries no
+// metrics block at all is a host running code from before the block existed —
+// which is a fact worth stating, not a row of zeros.
+func printLiveBoxes(w io.Writer, boxes []status.Box, now time.Time) {
+ sorted := append([]status.Box(nil), boxes...)
+ sort.Slice(sorted, func(i, j int) bool { return sorted[i].Label < sorted[j].Label })
+ for _, b := range sorted {
+ if status.BoxStale(b, now, status.SnapshotPeriod) {
+ fmt.Fprintf(w, "%-24s %-12s UNREACHABLE (last heartbeat %s ago)\n",
+ b.Label, orDash(b.Region), now.Sub(b.GeneratedAt).Round(time.Minute))
+ continue
+ }
+ m := b.Metrics
+ if m == nil {
+ fmt.Fprintf(w, "%-24s %-12s ok (no metrics block — pre-upgrade host)\n", b.Label, orDash(b.Region))
+ continue
+ }
+ fmt.Fprintf(w, "%-24s %-12s release=%s tunnel=%s mem=%s/%s load=%s disk-free=%s\n",
+ b.Label, orDash(b.Region), orDash(m.Release), orDash(m.Tunnel),
+ humanBytes(m.MemAvailBytes), humanBytes(m.MemTotalBytes),
+ humanFloat(m.Load1), humanBytes(m.DiskFreeBytes))
+ }
+}
+
+// orDash, humanBytes and humanFloat render one field of a metrics block.
+// Unknown is a dash in all three — never a zero, never a blank column: the
+// status package carries "I couldn't measure this" as a nil pointer all the
+// way to here precisely so the reader can tell it apart from a measured zero.
+func orDash(s string) string {
+ if s == "" {
+ return "-"
+ }
+ return s
+}
+
+func humanBytes(v *int64) string {
+ if v == nil {
+ return "-"
+ }
+ b := float64(*v)
+ switch {
+ case b >= 1<<30:
+ return fmt.Sprintf("%.1fGB", b/(1<<30))
+ case b >= 1<<20:
+ return fmt.Sprintf("%.1fMB", b/(1<<20))
+ case b >= 1<<10:
+ return fmt.Sprintf("%.1fKB", b/(1<<10))
+ default:
+ return fmt.Sprintf("%dB", *v)
+ }
+}
+
+func humanFloat(v *float64) string {
+ if v == nil {
+ return "-"
+ }
+ return fmt.Sprintf("%.2f", *v)
+}
diff --git a/internal/console/server.go b/internal/console/server.go
index e0031150a..e9bacda9b 100644
--- a/internal/console/server.go
+++ b/internal/console/server.go
@@ -97,6 +97,16 @@ var templates = template.Must(template.New("").Funcs(template.FuncMap{
return fmt.Sprintf("%d B", *v)
}
},
+ // loadOrDash renders a 1-minute load average: an absolute number of
+ // runnable tasks, not a fraction, so pct (which multiplies by 100 and
+ // stamps a %) would be a lie about the same *float64. nil is the same
+ // em-dash every unknown on this console gets.
+ "loadOrDash": func(v *float64) string {
+ if v == nil {
+ return "—"
+ }
+ return fmt.Sprintf("%.2f", *v)
+ },
// mapGet reads one named field out of Fleet.Instances — a map so a
// template range would iterate it key-sorted, alphabetically, which
// scrambles the static/resident/hibernating/awake/asleep order the
@@ -318,6 +328,14 @@ type page struct {
Fleet *status.Fleet // /fleet: nil means no snapshot published yet
+ // /fleet, below the published boxes table: the per-host PRIVATE detail
+ // read straight out of status/boxes/ (HostRows, view.go). Raw numbers
+ // render here and nowhere public — this page is session-gated to
+ // operators, fleet.json is not. Nil renders no section at all: no boxes
+ // yet, no bucket handle, or a boxes read that failed are all "no detail
+ // to show", never a reason to fail the page.
+ Hosts []HostRow
+
// /fleet/metrics: the operator dashboard (Task 11), both from LoadMetrics
// (metricsview.go) and both nil-safe to render — no boxes and no firing
// alerts are the normal, healthy, empty state, not an error.
@@ -1030,7 +1048,16 @@ func (s *Server) handleFleet(w http.ResponseWriter, r *http.Request) {
s.renderMessage(w, r, http.StatusInternalServerError, "Error", "Could not read fleet status. Please try again.")
return
}
- p := page{Title: "Fleet status", Address: addr, CSRF: raw, Fleet: f}
+ // The private host detail is an EXTRA read on a page that already has its
+ // answer: a bucket that won't list the boxes costs the operator the
+ // per-host block, not the whole fleet page, so this degrades with a log
+ // line rather than turning a readable page into a 500.
+ hosts, err := HostRows(r.Context(), s.cfg.Releases, time.Now())
+ if err != nil {
+ log.Printf("console: fleet hosts: %v", err)
+ hosts = nil
+ }
+ p := page{Title: "Fleet status", Address: addr, CSRF: raw, Fleet: f, Hosts: hosts}
s.render(w, http.StatusOK, "fleet", p)
}
diff --git a/internal/console/server_test.go b/internal/console/server_test.go
index 99a94cb4d..9c5d054a9 100644
--- a/internal/console/server_test.go
+++ b/internal/console/server_test.go
@@ -1161,6 +1161,133 @@ func TestFleetKeylessReadable(t *testing.T) {
}
}
+// seedHostBoxes writes one fresh and one long-dark box snapshot, both
+// carrying a private Metrics block, and publishes the merged feed over them —
+// the exact pair /fleet's Hosts detail has to tell apart.
+func seedHostBoxes(t *testing.T, fx *consoleFixture) {
+ t.Helper()
+ ctx := context.Background()
+ now := time.Now()
+ for _, b := range []status.Box{
+ {
+ GeneratedAt: now, Label: "ada-stargazing", Region: "eu-west-1",
+ Metrics: &status.BoxMetrics{
+ MemTotalBytes: pi64(2 << 30), MemAvailBytes: pi64(1 << 30),
+ Load1: p(0.42), DiskFreeBytes: pi64(5 << 30),
+ Release: "abc1234", Tunnel: "connected",
+ },
+ },
+ {
+ GeneratedAt: now.Add(-time.Hour), Label: "grace-kayaking", Region: "eu-west-2",
+ Metrics: &status.BoxMetrics{
+ MemTotalBytes: pi64(9 << 30), MemAvailBytes: pi64(7 << 30),
+ Load1: p(8.25), DiskFreeBytes: pi64(3 << 30),
+ Release: "beefcaf", Tunnel: "connected",
+ },
+ },
+ } {
+ if err := status.WriteBox(ctx, fx.rel, b); err != nil {
+ t.Fatal(err)
+ }
+ }
+ f, err := status.BuildFleet(ctx, fx.rel, nil, now, status.SnapshotPeriod)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := status.PublishFleet(ctx, fx.rel, f); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// TestFleetHostsDetailSuppressesStaleMetrics is Task 7's core rule, on the one
+// surface allowed to render raw numbers at all: the operator fleet page shows
+// a live host's release, tunnel state, memory, load and free disk — and shows
+// an unreachable host as unreachable, with how long it has been dark and NOT
+// ONE of its last happy numbers. A dead box's final reading rendered as a
+// current figure is the failure this whole task exists to prevent.
+func TestFleetHostsDetailSuppressesStaleMetrics(t *testing.T) {
+ fx := newConsoleFixture(t)
+ seedHostBoxes(t, fx)
+ c := signInPaul(t, fx)
+ if err := fx.st.SetOperator("paul@keymail.dev", true); err != nil {
+ t.Fatal(err)
+ }
+ code, _, body := get(t, c, fx.ts.URL+"/fleet")
+ if code != 200 {
+ t.Fatalf("/fleet = %d: %s", code, body)
+ }
+ for _, want := range []string{
+ "Hosts", "ada-stargazing", "abc1234", "connected",
+ "1.0 GB", "2.0 GB", "0.42", "5.0 GB", // the fresh box's raw readings
+ "grace-kayaking", "unreachable",
+ } {
+ if !strings.Contains(body, want) {
+ t.Fatalf("/fleet hosts detail missing %q in:\n%s", want, body)
+ }
+ }
+ for _, leaked := range []string{"beefcaf", "8.25", "7.0 GB", "9.0 GB", "3.0 GB"} {
+ if strings.Contains(body, leaked) {
+ t.Fatalf("unreachable host's stale reading %q rendered — a dark box must show no numbers:\n%s", leaked, body)
+ }
+ }
+}
+
+// TestFleetHostsDegradeWhenUnreadable: the Hosts detail is an extra read on a
+// page that already had a job. If the boxes can't be listed, the page still
+// renders the merged feed it could read — metrics are never the reason /fleet
+// 500s.
+func TestFleetHostsDegradeWhenUnreadable(t *testing.T) {
+ fx := newConsoleFixture(t)
+ seedFleetJSON(t, fx)
+ fx.srv.cfg.Releases = listFailStore{fx.rel}
+ c := signInPaul(t, fx)
+ if err := fx.st.SetOperator("paul@keymail.dev", true); err != nil {
+ t.Fatal(err)
+ }
+ code, _, body := get(t, c, fx.ts.URL+"/fleet")
+ if code != 200 || !strings.Contains(body, "Fleet status") || !strings.Contains(body, "ada-stargazing") {
+ t.Fatalf("/fleet with an unlistable bucket = %d, want the fleet feed still rendered:\n%s", code, body)
+ }
+}
+
+// TestFleetHostsEscapesHostSuppliedStrings: Release and Tunnel are strings a
+// BOX writes into its own snapshot — the console reads them, it does not
+// mint them — so a compromised or buggy host must not be able to put markup
+// on an operator's page. html/template escapes them because they are plain
+// strings all the way through (never template.HTML, unlike icons.go's
+// vendored SVG); this test is what keeps that true.
+func TestFleetHostsEscapesHostSuppliedStrings(t *testing.T) {
+ fx := newConsoleFixture(t)
+ if err := status.WriteBox(context.Background(), fx.rel, status.Box{
+ GeneratedAt: time.Now(), Label: "leo-juggling", Region: "eu-west-1",
+ Metrics: &status.BoxMetrics{Release: "<script>alert(1)</script>", Tunnel: "connected"},
+ }); err != nil {
+ t.Fatal(err)
+ }
+ c := signInPaul(t, fx)
+ if err := fx.st.SetOperator("paul@keymail.dev", true); err != nil {
+ t.Fatal(err)
+ }
+ code, _, body := get(t, c, fx.ts.URL+"/fleet")
+ if code != 200 {
+ t.Fatalf("/fleet = %d: %s", code, body)
+ }
+ if strings.Contains(body, "<script>alert(1)</script>") {
+ t.Fatalf("a host-supplied release string reached the page unescaped:\n%s", body)
+ }
+ if !strings.Contains(body, "&lt;script&gt;alert(1)&lt;/script&gt;") {
+ t.Fatalf("escaped release string not rendered at all:\n%s", body)
+ }
+}
+
+// listFailStore is a store whose List always fails — everything else passes
+// through, so fleet.json still reads and only the boxes sweep breaks.
+type listFailStore struct{ release.Store }
+
+func (listFailStore) List(ctx context.Context, prefix string) ([]string, error) {
+ return nil, errors.New("bucket list refused")
+}
+
// TestFleetMetricsOperatorOnly is Task 11's gate, mirroring TestFleetOperatorOnly
// for /fleet/metrics: signed out and signed in-but-not-operator both get
// exactly the same 404 an unrouted URL gets; an operator gets 200 with a
diff --git a/internal/console/templates/fleet.html b/internal/console/templates/fleet.html
index f82bcf944..f125a2dc4 100644
--- a/internal/console/templates/fleet.html
+++ b/internal/console/templates/fleet.html
@@ -91,4 +91,40 @@
{{else}}
<p>No snapshots yet — the fleet status writer hasn't published <code>status/fleet.json</code> on this deployment.</p>
{{end}}
+
+{{if .Hosts}}
+<h2>Hosts</h2>
+<div class="tablewrap">
+<table>
+ <tr>
+ <th>Host</th><th>Region</th><th>Release</th><th>Tunnel</th>
+ <th>Memory free</th><th>Load</th><th>Disk free</th><th>Last report</th>
+ </tr>
+ {{range .Hosts}}
+ <tr>
+ <td>{{.Label}}</td>
+ <td>{{emptyOrDash .Region}}</td>
+ {{if .Metrics}}
+ <td>{{emptyOrDash .Metrics.Release}}</td>
+ <td>{{emptyOrDash .Metrics.Tunnel}}</td>
+ <td>{{bytesOrDash .Metrics.MemAvailBytes}} / {{bytesOrDash .Metrics.MemTotalBytes}}</td>
+ <td>{{loadOrDash .Metrics.Load1}}</td>
+ <td>{{bytesOrDash .Metrics.DiskFreeBytes}}</td>
+ <td>{{fmtTime .LastSeen}}</td>
+ {{else if .Stale}}
+ {{/* An unreachable host says how long it has been dark and nothing else:
+ its last readings are history, and a history rendered in a "now"
+ column is how an outage gets missed. */}}
+ <td colspan="5"><span class="chip chip-down">unreachable</span> last heartbeat {{.Dark}} ago</td>
+ <td>{{fmtTime .LastSeen}}</td>
+ {{else}}
+ <td colspan="5" class="small">no metrics block — this host predates raw metric reporting</td>
+ <td>{{fmtTime .LastSeen}}</td>
+ {{end}}
+ </tr>
+ {{end}}
+</table>
+</div>
+<p class="small">Read from <code>status/boxes/</code> in the deployment bucket, which is not public — these are raw figures, unlike the aggregate feed above, and this page is why they stay behind a session.</p>
+{{end}}
{{template "bottom" .}}{{end}}
diff --git a/internal/console/view.go b/internal/console/view.go
index 2f34e51ed..25aab922a 100644
--- a/internal/console/view.go
+++ b/internal/console/view.go
@@ -11,6 +11,7 @@ import (
"errors"
"os"
"sort"
+ "strconv"
"strings"
"sync"
"time"
@@ -447,6 +448,89 @@ func FleetView(ctx context.Context, s release.Store) (*status.Fleet, error) {
return &f, nil
}
+// HostRow is one box's PRIVATE detail line on the operator fleet page: the
+// raw block status.Box carries and fleet.json deliberately never publishes
+// (internal/status's package comment), plus whether the box is still
+// reporting at all.
+//
+// Metrics is nil for a box that is STALE as well as for one that never
+// reported any — suppression happens here, in the view model, not in the
+// template. That is the point: an unreachable host's last happy numbers must
+// never reach the page, and the only way to be sure of that is for the page
+// to never be handed them. A template condition would be one careless
+// {{if}}-rearrangement away from printing a dead box's memory as a current
+// figure.
+type HostRow struct {
+ Label string
+ Region string
+ // Stale is status.BoxStale — the fleet's one staleness rule, measured
+ // against status.SnapshotPeriod, so this table and the published boxes
+ // table above it can never disagree about which host is dark.
+ Stale bool
+ // LastSeen is the snapshot's own GeneratedAt and Dark is how long ago that
+ // was, pre-rendered ("1h2m"). Dark is set only for a stale row: it is the
+ // one thing an unreachable host is allowed to say about itself.
+ LastSeen time.Time
+ Dark string
+ Metrics *status.BoxMetrics
+}
+
+// HostRows reads every per-box snapshot (status.LoadBoxes — the same loader
+// and the same skip-corrupt rule the public merge uses) and turns it into the
+// operator page's private host table, sorted by label so the order is stable
+// across refreshes.
+//
+// A nil store is not an error: a deployment configured without a bucket
+// handle simply has no host detail to show, and the fleet page renders
+// without the section rather than failing. Same posture for the caller on a
+// read error — see handleFleet.
+func HostRows(ctx context.Context, s release.Store, now time.Time) ([]HostRow, error) {
+ if s == nil {
+ return nil, nil
+ }
+ boxes, err := status.LoadBoxes(ctx, s)
+ if err != nil {
+ return nil, err
+ }
+ rows := make([]HostRow, 0, len(boxes))
+ for _, b := range boxes {
+ row := HostRow{
+ Label: b.Label,
+ Region: b.Region,
+ Stale: status.BoxStale(b, now, status.SnapshotPeriod),
+ LastSeen: b.GeneratedAt,
+ Metrics: b.Metrics,
+ }
+ if row.Stale {
+ row.Dark = humanAge(now.Sub(b.GeneratedAt))
+ row.Metrics = nil // an unreachable host shows no numbers at all
+ }
+ rows = append(rows, row)
+ }
+ sort.Slice(rows, func(i, j int) bool { return rows[i].Label < rows[j].Label })
+ return rows, nil
+}
+
+// humanAge renders how long a box has been dark at the resolution an operator
+// actually reads it at: minutes up to an hour, then hours and minutes, then
+// days. Never seconds — a snapshot is only ever called stale after three
+// missed five-minute ticks, so second-level precision would be false detail —
+// and never a negative age: a box whose clock runs ahead of the console's
+// reads as "0m", not as a time traveller.
+func humanAge(d time.Duration) string {
+ if d < 0 {
+ d = 0
+ }
+ switch {
+ case d < time.Hour:
+ return strconv.Itoa(int(d.Minutes())) + "m"
+ case d < 24*time.Hour:
+ return strconv.Itoa(int(d.Hours())) + "h" + strconv.Itoa(int(d.Minutes())%60) + "m"
+ default:
+ return strconv.Itoa(int(d.Hours())/24) + "d" + strconv.Itoa(int(d.Hours())%24) + "h"
+ }
+}
+
// versionsCap bounds the app page's release list.
const versionsCap = 20
diff --git a/internal/host/agent.go b/internal/host/agent.go
index 8316313ca..09f968a6a 100644
--- a/internal/host/agent.go
+++ b/internal/host/agent.go
@@ -540,6 +540,16 @@ func domainsLoop(reg *registry.Registry, acmeCacheFlag string) {
// has other env-reading tests; a plain map/func seam avoids that coupling
// entirely.
func resolveStatusBucket(getenv func(string) string) string {
+ return ResolveStatusBucket(getenv)
+}
+
+// ResolveStatusBucket is resolveStatusBucket for callers outside this package
+// — `carlos status -live` reads the boxes an agent wrote, and it must land on
+// the SAME bucket the agent's status loop writes to, including the
+// CARLOS_DEPLOYMENT_BUCKET fallback. Exported as a second name over one
+// implementation rather than renamed at every internal call site, so the
+// defaulting rule documented above stays in exactly one function.
+func ResolveStatusBucket(getenv func(string) string) string {
if b := getenv("CARLOS_STATUS_BUCKET"); b != "" {
return b
}
@@ -707,8 +717,10 @@ func statusTick(ctx context.Context, s release.Store, reg *registry.Registry, p
return nil
}
- const fleetPeriod = 5 * time.Minute // matches statusLoop's own tick cadence
- fleet, err := status.BuildFleet(ctx, s, routes, now, fleetPeriod)
+ // status.SnapshotPeriod is this loop's own tick cadence, named once in the
+ // status package so the console's private host table and `carlos status
+ // -live` measure staleness against the same period this publish does.
+ fleet, err := status.BuildFleet(ctx, s, routes, now, status.SnapshotPeriod)
if err != nil {
return err
}
diff --git a/internal/status/fleet.go b/internal/status/fleet.go
index 7f69d8e92..5dd0c8938 100644
--- a/internal/status/fleet.go
+++ b/internal/status/fleet.go
@@ -95,9 +95,52 @@ var capacityRank = map[string]int{
// without this helper.
func intPtr(v int) *int { return &v }
+// SnapshotPeriod is how often every box republishes its own
+// status/boxes/<label>.json (the agent's status loop tick, 5 minutes +
+// jitter). It lives here rather than at each reader because it is the unit
+// BoxStale measures staleness in: the console's private host detail and
+// `carlos status -live` both have to answer "is this box dark?" without an
+// agent's local const to borrow, and a reader that guessed a different period
+// would call a box unreachable at a different moment than the published
+// boxes table does.
+const SnapshotPeriod = 5 * time.Minute
+
+// staleFactor is how many missed snapshot periods make a box stale. Three
+// consecutive misses: past any single hiccup (a slow S3 write, one skipped
+// tick), and the point at which a box has plainly stopped self-reporting.
+// One constant, one rule — BuildFleet's published FleetBox.Stale and every
+// private reader go through BoxStale below, so nothing can drift into calling
+// the same box fresh on one page and dark on another.
+const staleFactor = 3
+
+// BoxStale is the fleet's one staleness rule: a snapshot older than
+// staleFactor×period means the box stopped reporting. Every caller — the
+// public merge, the console's private host table, the CLI — asks this
+// function, never its own subtraction.
+//
+// Callers that render must treat true as "this box's numbers are history":
+// per the design spec an unreachable host shows how long it has been dark and
+// NOTHING else, never its last happy readings.
+func BoxStale(b Box, now time.Time, period time.Duration) bool {
+ return now.Sub(b.GeneratedAt) > staleFactor*period
+}
+
+// LoadBoxes reads every per-box snapshot in the bucket — the same read
+// BuildFleet does, exported for the private readers (the console's operator
+// fleet page, `carlos status -live`) so there is exactly ONE parser of the
+// status/boxes/ shape and one skip-corrupt rule, rather than a second loader
+// that could disagree about which snapshots count.
+//
+// Unlike the merged feed, what comes back here still carries Box.Metrics: raw
+// numbers that must reach a private surface only (bucket credentials or a
+// session-gated operator page), never fleet.json. See the package comment.
+func LoadBoxes(ctx context.Context, s release.Store) ([]Box, error) {
+ return loadBoxes(ctx, s)
+}
+
// BuildFleet merges every status/boxes/*.json (stale = GeneratedAt older
-// than 3×period), folds the local registry's routes into apps/services,
-// and never invents a number a snapshot didn't carry.
+// than staleFactor×period — see BoxStale), folds the local registry's routes
+// into apps/services, and never invents a number a snapshot didn't carry.
func BuildFleet(ctx context.Context, s release.Store, routes []registry.Route, now time.Time, period time.Duration) (*Fleet, error) {
boxes, err := loadBoxes(ctx, s)
if err != nil {
@@ -112,7 +155,7 @@ func BuildFleet(ctx context.Context, s release.Store, routes []registry.Route, n
// fresh boxes.
stale := make([]bool, len(boxes))
for i, b := range boxes {
- stale[i] = now.Sub(b.GeneratedAt) > 3*period
+ stale[i] = BoxStale(b, now, period)
f.Boxes = append(f.Boxes, FleetBox{
Label: b.Label,
Region: b.Region,
diff --git a/internal/status/status_test.go b/internal/status/status_test.go
index b3d907a25..142414b80 100644
--- a/internal/status/status_test.go
+++ b/internal/status/status_test.go
@@ -820,3 +820,77 @@ func TestFleetBoxCarriesAdoption(t *testing.T) {
f.Boxes[0].AdoptionRunning)
}
}
+
+// ---- Task 7: the exported box loader and the one staleness rule -------------
+
+// TestLoadBoxesAndStaleness pins the two things the console's private host
+// detail and `carlos status -live` both read the bucket through: LoadBoxes
+// returns every snapshot WITH its private Metrics block intact (the merged
+// public feed drops it — this loader must not), and BoxStale answers the same
+// "older than 3 periods" question BuildFleet's own boxes table answers, so the
+// two renderings can never disagree about which host is dark.
+func TestLoadBoxesAndStaleness(t *testing.T) {
+ s := teststore(t)
+ ctx := context.Background()
+ now := time.Now()
+ fresh := status.Box{GeneratedAt: now, Label: "ada-stargazing",
+ Metrics: &status.BoxMetrics{Release: "abc1234", Tunnel: "connected"}}
+ dark := status.Box{GeneratedAt: now.Add(-time.Hour), Label: "grace-kayaking"}
+ for _, b := range []status.Box{fresh, dark} {
+ if err := status.WriteBox(ctx, s, b); err != nil {
+ t.Fatal(err)
+ }
+ }
+ boxes, err := status.LoadBoxes(ctx, s)
+ if err != nil || len(boxes) != 2 {
+ t.Fatalf("LoadBoxes: %v, %d boxes", err, len(boxes))
+ }
+ for _, b := range boxes {
+ wantStale := b.Label == "grace-kayaking"
+ if got := status.BoxStale(b, now, 5*time.Minute); got != wantStale {
+ t.Fatalf("%s: stale = %v, want %v", b.Label, got, wantStale)
+ }
+ if b.Label == "ada-stargazing" {
+ if b.Metrics == nil || b.Metrics.Release != "abc1234" || b.Metrics.Tunnel != "connected" {
+ t.Fatalf("LoadBoxes dropped the private metrics block: %+v", b.Metrics)
+ }
+ }
+ }
+}
+
+// TestBoxStaleMatchesFleetBoxStale is the anti-drift assertion: whatever
+// BoxStale says about a box must be exactly what BuildFleet publishes in that
+// box's FleetBox.Stale — one rule, one constant, two renderings.
+func TestBoxStaleMatchesFleetBoxStale(t *testing.T) {
+ s := teststore(t)
+ ctx := context.Background()
+ now := time.Now()
+ const period = 5 * time.Minute
+ for _, b := range []status.Box{
+ boxFixture("just-now", "plenty", now),
+ boxFixture("two-periods", "plenty", now.Add(-2*period)),
+ boxFixture("exactly-three", "plenty", now.Add(-3*period)),
+ boxFixture("long-dark", "plenty", now.Add(-time.Hour)),
+ } {
+ if err := status.WriteBox(ctx, s, b); err != nil {
+ t.Fatal(err)
+ }
+ }
+ f, err := status.BuildFleet(ctx, s, nil, now, period)
+ if err != nil {
+ t.Fatal(err)
+ }
+ boxes, err := status.LoadBoxes(ctx, s)
+ if err != nil {
+ t.Fatal(err)
+ }
+ byLabel := map[string]bool{}
+ for _, fb := range f.Boxes {
+ byLabel[fb.Label] = fb.Stale
+ }
+ for _, b := range boxes {
+ if got, want := status.BoxStale(b, now, period), byLabel[b.Label]; got != want {
+ t.Errorf("%s: BoxStale = %v, FleetBox.Stale = %v — the rule has drifted", b.Label, got, want)
+ }
+ }
+}
diff --git a/statuslive_test.go b/statuslive_test.go
new file mode 100644
index 000000000..028bae66d
--- /dev/null
+++ b/statuslive_test.go
@@ -0,0 +1,127 @@
+package main
+
+import (
+ "bytes"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/status"
+)
+
+func f64(v float64) *float64 { return &v }
+func i64(v int64) *int64 { return &v }
+
+// TestPrintLiveBoxesSuppressesStaleMetrics is the CLI half of the rule the
+// console page enforces: a host that stopped reporting prints as UNREACHABLE
+// with how long it has been dark, and NONE of its last readings. The operator
+// running this has bucket credentials, so the numbers are theirs to see — the
+// point is that a dead box's numbers are not a current answer to anything.
+func TestPrintLiveBoxesSuppressesStaleMetrics(t *testing.T) {
+ now := time.Now()
+ boxes := []status.Box{
+ {
+ GeneratedAt: now, Label: "ada-stargazing", Region: "eu-west-1",
+ Metrics: &status.BoxMetrics{
+ MemTotalBytes: i64(2 << 30), MemAvailBytes: i64(1 << 30),
+ Load1: f64(0.42), DiskFreeBytes: i64(5 << 30),
+ Release: "abc1234", Tunnel: "connected",
+ },
+ },
+ {
+ GeneratedAt: now.Add(-90 * time.Minute), Label: "grace-kayaking", Region: "eu-west-2",
+ Metrics: &status.BoxMetrics{
+ MemTotalBytes: i64(9 << 30), MemAvailBytes: i64(7 << 30),
+ Load1: f64(8.25), DiskFreeBytes: i64(3 << 30),
+ Release: "beefcaf", Tunnel: "connected",
+ },
+ },
+ }
+ var buf bytes.Buffer
+ printLiveBoxes(&buf, boxes, now)
+ out := buf.String()
+
+ for _, want := range []string{
+ "ada-stargazing", "eu-west-1", "release=abc1234", "tunnel=connected",
+ "mem=1.0GB/2.0GB", "load=0.42", "disk-free=5.0GB",
+ "grace-kayaking", "UNREACHABLE", "1h30m0s",
+ } {
+ if !strings.Contains(out, want) {
+ t.Fatalf("missing %q in:\n%s", want, out)
+ }
+ }
+ for _, leaked := range []string{"beefcaf", "8.25", "7.0GB", "9.0GB", "3.0GB"} {
+ if strings.Contains(out, leaked) {
+ t.Fatalf("stale host's reading %q printed:\n%s", leaked, out)
+ }
+ }
+}
+
+// TestPrintLiveBoxesPreUpgradeHost: a box running code older than the metrics
+// block is fresh and fine — it says so, in the same columns, rather than
+// printing a row of fabricated zeros or dropping out of the listing.
+func TestPrintLiveBoxesPreUpgradeHost(t *testing.T) {
+ now := time.Now()
+ var buf bytes.Buffer
+ printLiveBoxes(&buf, []status.Box{
+ {GeneratedAt: now, Label: "alan-foraging"}, // no metrics, no region
+ {GeneratedAt: now, Label: "iris-sailing", Metrics: &status.BoxMetrics{Tunnel: "reconnecting"}},
+ }, now)
+ out := buf.String()
+ if !strings.Contains(out, "alan-foraging") || !strings.Contains(out, "no metrics block") {
+ t.Fatalf("pre-upgrade host row wrong:\n%s", out)
+ }
+ // A metrics block with only some fields known dashes the rest — never a
+ // zero, the same "unknown is not a number" rule the status package holds.
+ if !strings.Contains(out, "tunnel=reconnecting") || !strings.Contains(out, "mem=-/-") ||
+ !strings.Contains(out, "load=-") || !strings.Contains(out, "disk-free=-") ||
+ !strings.Contains(out, "release=-") {
+ t.Fatalf("partial metrics row wrong:\n%s", out)
+ }
+ for _, line := range strings.Split(strings.TrimRight(out, "\n"), "\n") {
+ if !strings.HasPrefix(line, "alan-foraging") && !strings.HasPrefix(line, "iris-sailing") {
+ t.Fatalf("row does not start with its label (alignment broken):\n%s", out)
+ }
+ }
+}
+
+// TestPrintLiveBoxesSortsByLabel: the listing is stable across runs, so a
+// diff between two invocations is a change in the fleet, not in list order.
+func TestPrintLiveBoxesSortsByLabel(t *testing.T) {
+ now := time.Now()
+ var buf bytes.Buffer
+ printLiveBoxes(&buf, []status.Box{
+ {GeneratedAt: now, Label: "zoe-welding"},
+ {GeneratedAt: now, Label: "ada-stargazing"},
+ }, now)
+ if a, z := strings.Index(buf.String(), "ada-"), strings.Index(buf.String(), "zoe-"); a > z {
+ t.Fatalf("rows are not label-sorted:\n%s", buf.String())
+ }
+}
+
+func TestHumanBytesAndFloat(t *testing.T) {
+ for _, c := range []struct {
+ in *int64
+ want string
+ }{
+ {nil, "-"},
+ {i64(0), "0B"},
+ {i64(512), "512B"},
+ {i64(1536), "1.5KB"},
+ {i64(3 << 20), "3.0MB"},
+ {i64(2 << 30), "2.0GB"},
+ } {
+ if got := humanBytes(c.in); got != c.want {
+ t.Errorf("humanBytes(%v) = %q, want %q", c.in, got, c.want)
+ }
+ }
+ if got := humanFloat(nil); got != "-" {
+ t.Errorf("humanFloat(nil) = %q, want -", got)
+ }
+ if got := humanFloat(f64(1.5)); got != "1.50" {
+ t.Errorf("humanFloat(1.5) = %q, want 1.50", got)
+ }
+ if got := orDash(""); got != "-" {
+ t.Errorf("orDash(\"\") = %q, want -", got)
+ }
+}