instances, routes, store: show database and bucket sizes
Each box now stats every route's SQLite database (file plus -wal) on its status tick and publishes host -> bytes in the private status/boxes Metrics block (route_db_bytes, additive, omitted when empty). The console joins it into `carlos instances list` and `carlos routes` as db_bytes, under the same staleness rule as staged_version: a stale or silent box contributes nothing, and an unmeasured host is absent, never 0. `carlos store status` now shows the bucket's size from the console's existing daily storage sweep, dated. It does not list the bucket per call. A sweep dated before the grant is skipped so a fresh store does not read as empty, and an unswept day prints as a warning rather than a size.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
16 files changed,
+426
−11
apimode.go+16 −0docs/site/cli.md+15 −0instances_cli.go+2 −2internal/console/apiinstances.go+24 −2internal/console/apiinstances_test.go+49 −0internal/console/apiroutes.go+6 −1internal/console/apistore.go+66 −0internal/console/apistore_test.go+60 −0internal/host/agent.go+3 −0internal/host/metrics_probes_test.go+26 −0internal/host/probes.go+20 −0internal/status/snapshot.go+29 −2internal/status/status_test.go+55 −0routes_cli.go+13 −4store.go+16 −0store_test.go+26 −0
CI passed — run details
diff --git a/apimode.go b/apimode.go| index 5b7b0cec7..16adcf4c7 100644 |
| --- a/apimode.go |
| +++ b/apimode.go |
| @@ -1657,6 +1657,17 @@ type apiStoreReply struct { |
| // exactly as it always did. |
| Scanning *apiScanBlock `json:"scanning,omitempty"` |
| Entitlement *objstore.Entitlement `json:"entitlement"` |
| + // Size is the bucket's size from the console's daily storage sweep, nil |
| + // before a sweep has measured it (and from an older console). |
| + Size *apiStoreSize `json:"size"` |
| +} |
| + |
| +// apiStoreSize mirrors the console's storeSizeReply (internal/console/apistore.go). |
| +type apiStoreSize struct { |
| + AsOf string `json:"as_of"` |
| + StoreBytes int64 `json:"store_bytes"` |
| + Unswept bool `json:"unswept,omitempty"` |
| + BucketBytes int64 `json:"bucket_bytes"` |
| } |
| const storeProvisionHTTPTimeout = 75 * time.Second |
| @@ -2857,6 +2868,10 @@ type apiInstanceRecord struct { |
| // column can give, since the person reading it is checking whether |
| // something private is exposed. |
| Gated *bool `json:"gated"` |
| + // DBBytes is the instance's SQLite database size as its box last |
| + // measured it (instanceListRecord.DBBytes). nil when unknown, or from a |
| + // console that predates the field — the DB column prints a dash. |
| + DBBytes *int64 `json:"db_bytes,omitempty"` |
| } |
| // apiUpstreamState mirrors status.UpstreamState's wire shape (internal/status) |
| @@ -3122,6 +3137,7 @@ type apiRouteRow struct { |
| Backing string `json:"backing"` |
| Provisioned bool `json:"provisioned"` |
| Replication string `json:"replication"` |
| + DBBytes *int64 `json:"db_bytes,omitempty"` // nil = unknown, printed as a dash |
| } |
| // apiRoutePathRow is one path route of the same answer: a leading URL |
diff --git a/docs/site/cli.md b/docs/site/cli.md| index 6a731adb4..1f638d8fc 100644 |
| --- a/docs/site/cli.md |
| +++ b/docs/site/cli.md |
| @@ -520,6 +520,12 @@ carlos instances list --app hello |
| `not-responding` and the rest. That is usually what you want when something |
| is wrong. |
| +The `DB` column is the size of each instance's SQLite database on disk, |
| +including its write-ahead log, as its box last measured it. Boxes measure |
| +every few minutes, and asleep instances are measured too. A dash means no box |
| +has measured it: the instance is not running anywhere yet, it has not created |
| +its database, or it keeps its data somewhere CARLOS does not manage. |
| + |
| `--environment <e>` on `create`, and `set-environment` on a route that |
| already exists, bind one instance to one of the app's named config bundles. |
| This matters most for a route standing **alongside** a live one. Config is |
| @@ -630,6 +636,9 @@ carlos routes --app hello |
| It reads the console's own box, so on a multi-box fleet a short answer is not |
| necessarily a complete one. The command says so every time it prints. |
| +Its `DB` column is each route's SQLite database size, the same figure as in |
| +[carlos instances](/docs/cli#carlos-instances). |
| + |
| ### carlos domains |
| Attaches a customer's own hostname to one of the app's routes, detaches it, |
| @@ -782,6 +791,12 @@ after the cloud provider destroys it; during that destroy/re-create window, |
| retry after the provider releases the name. `status` shows an incomplete store |
| so you can tell this from an undeclared store. |
| +Once the store is provisioned, `status` also shows how big the bucket is. That |
| +figure comes from a sweep the console runs once a day, so it carries the date |
| +it was measured and can be up to a day old. Sizing a bucket means listing |
| +every object in it, and `status` does not do that on each call. A fresh store |
| +reads "not measured yet" until the next sweep. |
| + |
| `carlos store rotate` mints a fresh credential and leaves the old one live |
| until you run it again with `--finish`, so the app has a window to pick the |
| new one up. Rotating, declaring and reading status are member verbs and want |
diff --git a/instances_cli.go b/instances_cli.go| index ad4fdc5f9..5ffc16865 100644 |
| --- a/instances_cli.go |
| +++ b/instances_cli.go |
| @@ -370,13 +370,13 @@ func runInstancesList(args []string) { |
| log.Fatalf("instances list: %v", err) |
| } |
| w := newTable(stdout) |
| - fmt.Fprintln(w, "HOST\tREGION\tCHANNEL\tPOOL\tVERSION\tHEALTH\tUPSTREAMS\tCREATED\tBY") |
| + fmt.Fprintln(w, "HOST\tREGION\tCHANNEL\tPOOL\tVERSION\tHEALTH\tUPSTREAMS\tDB\tCREATED\tBY") |
| for _, r := range recs { |
| rowHealth := r.Health |
| if rowHealth == "" { |
| rowHealth = "—" // an old console: the field predates it, not "unhealthy" |
| } |
| - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", r.Host, r.Region, r.Channel, r.Pool, r.StagedVersion, rowHealth, upstreamsCell(r.Upstreams), r.CreatedAt, r.CreatedBy) |
| + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", r.Host, r.Region, r.Channel, r.Pool, r.StagedVersion, rowHealth, upstreamsCell(r.Upstreams), humanBytes(r.DBBytes), r.CreatedAt, r.CreatedBy) |
| } |
| w.Flush() |
| if truncated { |
diff --git a/internal/console/apiinstances.go b/internal/console/apiinstances.go| index b85fe8b8e..1256c0024 100644 |
| --- a/internal/console/apiinstances.go |
| +++ b/internal/console/apiinstances.go |
| @@ -184,6 +184,12 @@ type instanceListRecord struct { |
| // is always "" and omitted. Gated is what a member may know. |
| HiddenGate string `json:"gate,omitempty"` |
| Gated bool `json:"gated"` |
| + // DBBytes is the on-disk size of this instance's SQLite database (file |
| + // plus -wal) as its box last measured it (status.BoxMetrics.RouteDBBytes), |
| + // under the same staleness rule as StagedVersion. Absent when no fresh |
| + // box has sized it: not minted yet, no database file yet, or a route |
| + // whose database lives somewhere the platform does not know about. |
| + DBBytes *int64 `json:"db_bytes,omitempty"` |
| } |
| // instanceHealth computes each record host's health word the same way the |
| @@ -321,6 +327,18 @@ func (s *Server) healthByHost(ctx context.Context, routeFor map[string]registry. |
| type boxJoin struct { |
| versions map[string]string |
| upstreams map[string][]status.UpstreamState |
| + dbBytes map[string]int64 |
| +} |
| + |
| +// dbSize is the join's database size for host, nil when no fresh box |
| +// reported one — so the wire field is absent rather than a 0 that reads as |
| +// an empty database. |
| +func (j boxJoin) dbSize(host string) *int64 { |
| + n, ok := j.dbBytes[host] |
| + if !ok { |
| + return nil |
| + } |
| + return &n |
| } |
| // loadBoxJoin builds boxJoin from every NON-STALE box's own |
| @@ -356,7 +374,8 @@ func loadBoxJoin(ctx context.Context, s release.Store, now time.Time) boxJoin { |
| log.Printf("console: api instances: could not load status boxes for the list join: %v", err) |
| return boxJoin{} |
| } |
| - j := boxJoin{versions: map[string]string{}, upstreams: map[string][]status.UpstreamState{}} |
| + j := boxJoin{versions: map[string]string{}, upstreams: map[string][]status.UpstreamState{}, |
| + dbBytes: map[string]int64{}} |
| for _, b := range boxes { |
| if b.Metrics == nil || status.BoxStale(b, now, status.SnapshotPeriod) { |
| continue |
| @@ -367,6 +386,9 @@ func loadBoxJoin(ctx context.Context, s release.Store, now time.Time) boxJoin { |
| for host, ups := range b.Metrics.Upstreams { |
| j.upstreams[host] = ups |
| } |
| + for host, n := range b.Metrics.RouteDBBytes { |
| + j.dbBytes[host] = n |
| + } |
| } |
| return j |
| } |
| @@ -428,7 +450,7 @@ func (s *Server) handleAPIInstanceList(w http.ResponseWriter, r *http.Request) { |
| } |
| matched = append(matched, instanceListRecord{Record: rec, |
| StagedVersion: join.versions[rec.Host], Health: h, Upstreams: join.upstreams[rec.Host], |
| - ServedBy: served[rec.Host], Gated: rec.Gate != ""}) |
| + ServedBy: served[rec.Host], Gated: rec.Gate != "", DBBytes: join.dbSize(rec.Host)}) |
| } |
| total := len(matched) |
| // provision.ListRecords returns records host-sorted and nothing above |
diff --git a/internal/console/apiinstances_test.go b/internal/console/apiinstances_test.go| index 6c767d9fb..2c02e9063 100644 |
| --- a/internal/console/apiinstances_test.go |
| +++ b/internal/console/apiinstances_test.go |
| @@ -2556,3 +2556,52 @@ func TestAPIInstanceUpdateGateRefusals(t *testing.T) { |
| t.Fatalf("empty-body refusal should name gate: %s", out) |
| } |
| } |
| + |
| +// TestAPIInstanceListDBBytes: db_bytes joins from a FRESH box's |
| +// Metrics.RouteDBBytes, is absent (not 0) for a host no box sized, and is |
| +// dropped for a host whose only reporting box has gone stale — the same |
| +// rules as staged_version. |
| +func TestAPIInstanceListDBBytes(t *testing.T) { |
| + fx, token := instanceFixture(t, "code-inst-dbbytes") |
| + enableInstances(t, fx, "titogo", titoManifest()) |
| + u := instancesURL(fx, "bab", "titogo") |
| + for _, h := range []string{"acme.go.tito.io", "beta.go.tito.io", "gamma.go.tito.io"} { |
| + if code, _, out := rawReq(t, http.MethodPost, u, token, |
| + fmt.Sprintf(`{"host":%q,"region":"ie"}`, h)); code != http.StatusCreated { |
| + t.Fatalf("create %s = %d %s", h, code, out) |
| + } |
| + } |
| + fresh := status.CollectBox(nil, status.Probes{MachineID: "box1"}, time.Now()) |
| + fresh.Metrics = &status.BoxMetrics{RouteDBBytes: map[string]int64{"acme.go.tito.io": 123456}} |
| + if err := status.WriteBox(context.Background(), fx.rel, fresh); err != nil { |
| + t.Fatal(err) |
| + } |
| + stale := status.CollectBox(nil, status.Probes{MachineID: "box2"}, time.Now().Add(-time.Hour)) |
| + stale.Metrics = &status.BoxMetrics{RouteDBBytes: map[string]int64{"gamma.go.tito.io": 999}} |
| + if err := status.WriteBox(context.Background(), fx.rel, stale); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + code, _, out := rawReq(t, http.MethodGet, u, token, "") |
| + if code != http.StatusOK { |
| + t.Fatalf("list = %d %s", code, out) |
| + } |
| + var list struct { |
| + Instances []map[string]any `json:"instances"` |
| + } |
| + if err := json.Unmarshal([]byte(out), &list); err != nil { |
| + t.Fatalf("list body %s: %v", out, err) |
| + } |
| + got := map[string]map[string]any{} |
| + for _, r := range list.Instances { |
| + got[r["host"].(string)] = r |
| + } |
| + if v := got["acme.go.tito.io"]["db_bytes"]; v != float64(123456) { |
| + t.Errorf("acme db_bytes = %v, want 123456", v) |
| + } |
| + for _, h := range []string{"beta.go.tito.io", "gamma.go.tito.io"} { |
| + if v, ok := got[h]["db_bytes"]; ok { |
| + t.Errorf("%s db_bytes = %v, want the key absent", h, v) |
| + } |
| + } |
| +} |
diff --git a/internal/console/apiroutes.go b/internal/console/apiroutes.go| index bf16ca8dc..5dcd3f5ad 100644 |
| --- a/internal/console/apiroutes.go |
| +++ b/internal/console/apiroutes.go |
| @@ -67,6 +67,10 @@ type apiRoute struct { |
| // absent field as "", which is exactly what every route carried before |
| // there was a door onto the column (discussion #10). |
| Replication string `json:"replication,omitempty"` |
| + // DBBytes is the route's SQLite database size as its box last reported |
| + // it — the same box-snapshot join, and the same staleness rule, as the |
| + // instance list's db_bytes. Absent when unknown, never 0. |
| + DBBytes *int64 `json:"db_bytes,omitempty"` |
| } |
| // apiRoutePath is one path route: a leading URL segment on Host that is |
| @@ -144,6 +148,7 @@ func (s *Server) handleAPIRoutes(w http.ResponseWriter, r *http.Request) { |
| ctx, cancel := context.WithTimeout(r.Context(), routeListTimeout) |
| defer cancel() |
| health := s.healthByHost(ctx, routeFor) |
| + join := loadBoxJoin(ctx, s.cfg.Releases, time.Now()) |
| rows := make([]apiRoute, 0, len(routeFor)) |
| for _, rt := range routeFor { |
| rows = append(rows, apiRoute{ |
| @@ -151,7 +156,7 @@ func (s *Server) handleAPIRoutes(w http.ResponseWriter, r *http.Request) { |
| Channel: rt.Channel, Environment: rt.Environment, Status: rt.Status, |
| Health: health[rt.Host], Hibernate: rt.Hibernate, Unit: rt.Unit, |
| Backing: rt.Backing, Provisioned: rt.Provisioned, |
| - Replication: rt.Replication, |
| + Replication: rt.Replication, DBBytes: join.dbSize(rt.Host), |
| }) |
| } |
| // Sorted by host: registry order is not stable across passes, and a |
diff --git a/internal/console/apistore.go b/internal/console/apistore.go| index aeb419075..1db526b03 100644 |
| --- a/internal/console/apistore.go |
| +++ b/internal/console/apistore.go |
| @@ -35,16 +35,19 @@ package console |
| import ( |
| "context" |
| "crypto/ecdh" |
| + "encoding/json" |
| "errors" |
| "fmt" |
| "log" |
| "net/http" |
| + "os" |
| "strings" |
| "time" |
| "github.com/carlosframework/platform/internal/appconfig" |
| "github.com/carlosframework/platform/internal/objstore" |
| "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/usage" |
| ) |
| // storeTimeout bounds every store round trip these handlers make — the same |
| @@ -221,9 +224,72 @@ func (s *Server) storeReply(ctx context.Context, acct *Account, app string, rec |
| // would be a half-truth — but it is nil on every store that never |
| // opted in, which is all of them by default. |
| "scanning": rec.Scanning, |
| + // The bucket's size from the daily storage sweep, absent until a |
| + // sweep has measured this store. See storeSize. |
| + "size": s.storeSize(ctx, acct, app, rec), |
| }, nil |
| } |
| +// storeSizeLookback is how many UTC days storeSize walks back looking for a |
| +// storage sweep. The sweep runs on the console's first hourly economics wake |
| +// of each day, so today or yesterday normally answers; a week covers a |
| +// console that was down for a while, and anything older is too stale to |
| +// print as the bucket's size. |
| +const storeSizeLookback = 7 |
| + |
| +// storeSizeReply is the "size" block of `carlos store status`. It is copied |
| +// from the storage sweep (usage/storage/<date>.json) and is NOT measured on |
| +// this request: sizing a bucket means listing every object in it, which is |
| +// what the daily sweep already does once. Unswept means the sweep ran but |
| +// could not size this store (bad credential, endpoint down), and then |
| +// StoreBytes is meaningless. BucketBytes is the app's own files in the |
| +// deployment bucket (releases, config), a separate figure from its store. |
| +type storeSizeReply struct { |
| + AsOf string `json:"as_of"` |
| + StoreBytes int64 `json:"store_bytes"` |
| + Unswept bool `json:"unswept,omitempty"` |
| + BucketBytes int64 `json:"bucket_bytes"` |
| +} |
| + |
| +// storeSize finds the newest storage sweep from the last storeSizeLookback |
| +// days that has an answer for this app's store, or nil. A sweep that did not |
| +// see the store provisioned (it ran before the grant, or predates the |
| +// store_provisioned field) has no answer: its StoreBytes of 0 would read as |
| +// an empty bucket. Read failures are logged and answer nil; the size is an |
| +// annotation on status, never a reason for status to fail. |
| +func (s *Server) storeSize(ctx context.Context, acct *Account, app string, rec *objstore.Record) *storeSizeReply { |
| + if rec.Provisioned == nil { |
| + return nil |
| + } |
| + key := acct.Sqid + "/" + app |
| + now := time.Now() |
| + for i := 0; i < storeSizeLookback; i++ { |
| + date := usage.DayOf(now.AddDate(0, 0, -i)) |
| + raw, _, err := s.cfg.Releases.Get(ctx, usage.StorageDayKey(date)) |
| + if errors.Is(err, os.ErrNotExist) { |
| + continue |
| + } |
| + if err != nil { |
| + log.Printf("console: api store size %s: %v", usage.StorageDayKey(date), err) |
| + return nil |
| + } |
| + var day usage.StorageDay |
| + if err := json.Unmarshal(raw, &day); err != nil { |
| + log.Printf("console: api store size: parse %s: %v", usage.StorageDayKey(date), err) |
| + return nil |
| + } |
| + st, ok := day.Apps[key] |
| + if !ok || (!st.StoreProvisioned && !st.StoreUnswept) { |
| + // The newest sweep did not see this store; an older one cannot |
| + // have either. |
| + return nil |
| + } |
| + return &storeSizeReply{AsOf: date, StoreBytes: st.StoreBytes, |
| + Unswept: st.StoreUnswept, BucketBytes: st.BucketBytes} |
| + } |
| + return nil |
| +} |
| + |
| // handleAPIStoreCreate answers `carlos store create`: record that this app |
| // wants object storage, under a chosen (or default) env prefix. It is a |
| // REQUEST, not a provisioning — nothing exists in the backend when this |
diff --git a/internal/console/apistore_test.go b/internal/console/apistore_test.go| index 02c128e6e..d0a91832f 100644 |
| --- a/internal/console/apistore_test.go |
| +++ b/internal/console/apistore_test.go |
| @@ -11,6 +11,7 @@ package console |
| import ( |
| "context" |
| + "encoding/json" |
| "errors" |
| "net/http" |
| "strings" |
| @@ -20,6 +21,7 @@ import ( |
| "github.com/carlosframework/platform/internal/appconfig" |
| "github.com/carlosframework/platform/internal/objstore" |
| "github.com/carlosframework/platform/internal/release" |
| + "github.com/carlosframework/platform/internal/usage" |
| ) |
| func storeFixture(t *testing.T, code string) (*consoleFixture, string) { |
| @@ -1136,3 +1138,61 @@ func TestAPIStoreGrantFailureSaysResume(t *testing.T) { |
| t.Fatalf("buckets after the resume = %v", fake.Buckets) |
| } |
| } |
| + |
| +// TestAPIStoreStatusCarriesTheSweptSize: `size` is copied from the newest |
| +// storage sweep, dated. It is null when no sweep has measured the store, and |
| +// a sweep that did not see the store provisioned is never shown (its 0 would |
| +// read as an empty bucket). |
| +func TestAPIStoreStatusCarriesTheSweptSize(t *testing.T) { |
| + fx, token := storeFixture(t, "code-store-size") |
| + ctx := context.Background() |
| + now := time.Now() |
| + putRecord(t, fx, "titogo", &objstore.Record{ |
| + Declared: &objstore.Declared{EnvPrefix: "BLOBS", At: now, By: "alice@keymail.dev"}, |
| + Grant: &objstore.GrantBlock{Backend: "aws", At: now, By: "paul@keymail.dev"}, |
| + Provisioned: &objstore.Provisioned{Bucket: "carlos-bab-titogo", Region: "eu-west-1", |
| + IAMUser: "u", KeyID: "AKIA", At: now}, |
| + }) |
| + key := release.OpsAccount + "/titogo" |
| + seed := func(day time.Time, st usage.AppStorage) { |
| + t.Helper() |
| + body, _ := json.Marshal(usage.StorageDay{V: 1, Date: usage.DayOf(day), |
| + Apps: map[string]usage.AppStorage{key: st}}) |
| + if _, err := fx.rel.Put(ctx, usage.StorageDayKey(usage.DayOf(day)), body, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + size := func() map[string]any { |
| + t.Helper() |
| + code, out := getJSON(t, storeURL(fx, "titogo"), token) |
| + if code != http.StatusOK { |
| + t.Fatalf("status = %d %v", code, out) |
| + } |
| + if out["size"] == nil { |
| + return nil |
| + } |
| + return block(t, out, "size") |
| + } |
| + |
| + if got := size(); got != nil { |
| + t.Fatalf("size = %v with no sweep at all, want null", got) |
| + } |
| + |
| + // A sweep that ran before the grant: the store was not there to size. |
| + seed(now.AddDate(0, 0, -2), usage.AppStorage{BucketBytes: 10}) |
| + if got := size(); got != nil { |
| + t.Fatalf("size = %v from a sweep that did not see the store, want null", got) |
| + } |
| + |
| + seed(now.AddDate(0, 0, -1), usage.AppStorage{BucketBytes: 10, StoreBytes: 5 << 20, StoreProvisioned: true}) |
| + got := size() |
| + if got == nil || got["store_bytes"] != float64(5<<20) || got["as_of"] != usage.DayOf(now.AddDate(0, 0, -1)) { |
| + t.Fatalf("size = %v, want yesterday's 5MB", got) |
| + } |
| + |
| + // Today's sweep could not read the bucket: the newest answer says so. |
| + seed(now, usage.AppStorage{BucketBytes: 10, StoreUnswept: true}) |
| + if got := size(); got == nil || got["unswept"] != true || got["as_of"] != usage.DayOf(now) { |
| + t.Fatalf("size = %v, want today's unswept marker", got) |
| + } |
| +} |
diff --git a/internal/host/agent.go b/internal/host/agent.go| index 2b7f32e8a..af51537a3 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -2705,6 +2705,9 @@ func statusLoop(reg *registry.Registry) { |
| // edge-ready callback publishes the source, and on any box that |
| // probes nothing — both publish no key at all. |
| Upstreams: upstreamStates(), |
| + // One stat (two with a WAL) per database route per tick, which |
| + // is what `carlos instances list` and `carlos routes` show as DB. |
| + DBBytes: dbFileBytes, |
| } |
| // A fleet box (CARLOS_FLEET set) never gets here as a contender: it |
diff --git a/internal/host/metrics_probes_test.go b/internal/host/metrics_probes_test.go| index adae5520e..65100a32a 100644 |
| --- a/internal/host/metrics_probes_test.go |
| +++ b/internal/host/metrics_probes_test.go |
| @@ -237,3 +237,29 @@ func TestListCarlosUnitsFindsUnitsUnderCarlosSlice(t *testing.T) { |
| t.Fatalf("carlos-edge.service missing from the combined scan: %+v", combined) |
| } |
| } |
| + |
| +// TestDBFileBytes: the main file plus its -wal, nil (unknown) for a database |
| +// that does not exist, and the main file alone once a checkpoint has removed |
| +// the WAL. |
| +func TestDBFileBytes(t *testing.T) { |
| + dir := t.TempDir() |
| + db := filepath.Join(dir, "h.test.db") |
| + if got := dbFileBytes(db); got != nil { |
| + t.Fatalf("missing db = %d, want nil", *got) |
| + } |
| + if err := os.WriteFile(db, make([]byte, 4096), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + if got := dbFileBytes(db); got == nil || *got != 4096 { |
| + t.Fatalf("db alone = %v, want 4096", got) |
| + } |
| + if err := os.WriteFile(db+"-wal", make([]byte, 1000), 0o600); err != nil { |
| + t.Fatal(err) |
| + } |
| + if got := dbFileBytes(db); got == nil || *got != 5096 { |
| + t.Fatalf("db + wal = %v, want 5096", got) |
| + } |
| + if got := dbFileBytes(dir); got != nil { |
| + t.Fatalf("a directory = %d, want nil", *got) |
| + } |
| +} |
diff --git a/internal/host/probes.go b/internal/host/probes.go| index c17681403..d9e889387 100644 |
| --- a/internal/host/probes.go |
| +++ b/internal/host/probes.go |
| @@ -376,3 +376,23 @@ func readDiskFreeBytes() *int64 { |
| freeBytes := int64(avail) |
| return &freeBytes |
| } |
| + |
| +// dbFileBytes is status.Probes.DBBytes: the on-disk size of a route's SQLite |
| +// database, the main file plus its -wal. The WAL counts because in WAL mode |
| +// recent writes live there until a checkpoint, so the main file alone |
| +// understates a busy database. A missing or unreadable main file is nil |
| +// (unknown), never 0: a route that has not created its database yet has no |
| +// size. A missing -wal is normal after a checkpoint and adds nothing. A |
| +// hibernating route's file stays on disk while it sleeps (the activator keeps |
| +// it as a warm cache), so asleep routes are sized too. |
| +func dbFileBytes(dbPath string) *int64 { |
| + fi, err := os.Stat(dbPath) |
| + if err != nil || !fi.Mode().IsRegular() { |
| + return nil |
| + } |
| + n := fi.Size() |
| + if wal, err := os.Stat(dbPath + "-wal"); err == nil && wal.Mode().IsRegular() { |
| + n += wal.Size() |
| + } |
| + return &n |
| +} |
diff --git a/internal/status/snapshot.go b/internal/status/snapshot.go| index 5f1de396d..dd757d7dd 100644 |
| --- a/internal/status/snapshot.go |
| +++ b/internal/status/snapshot.go |
| @@ -123,6 +123,13 @@ type Probes struct { |
| // here means "no aliases known this tick," identical on the wire to "no |
| // alias rows exist." |
| Aliases []registry.RouteAlias |
| + |
| + // DBBytes sizes one route's SQLite database on local disk: the file at |
| + // registry.Route.DBPath plus its -wal. CollectBox calls it for every |
| + // non-platform, non-static route with a DBPath. A nil func, or a nil |
| + // answer (no file yet, stat failed), means unknown and the host is left |
| + // out of Metrics.RouteDBBytes rather than published as 0. |
| + DBBytes func(dbPath string) *int64 |
| } |
| // UpstreamState is one upstream's health as the edge's prober currently sees |
| @@ -234,6 +241,13 @@ type BoxMetrics struct { |
| // present-but-empty map) when this box has no alias rows, same |
| // "absent, not empty" rule as RouteVersions. |
| Aliases map[string]AliasReading `json:"aliases,omitempty"` |
| + // RouteDBBytes is host -> on-disk size of that route's SQLite database |
| + // (file plus -wal), for this box's own routes with a DBPath. It is |
| + // RouteVersions' twin: hostname-keyed, so it lives in this private block |
| + // and nowhere else. The console's instance and route listings are the |
| + // readers. A host whose file could not be sized is absent, never 0; the |
| + // whole map is omitted when no route could be sized. |
| + RouteDBBytes map[string]int64 `json:"route_db_bytes,omitempty"` |
| } |
| // AliasReading is one alias hostname's status as CollectBox saw it this |
| @@ -458,6 +472,9 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box { |
| // both fold into the same map — nil (not an empty map) until the first |
| // entry, same "nothing to report" convention as routeVersions. |
| var aliasReadings map[string]AliasReading |
| + // routeDBBytes is Metrics.RouteDBBytes, built in the same loop and nil |
| + // until the first entry, like routeVersions. |
| + var routeDBBytes map[string]int64 |
| for _, r := range routes { |
| // Platform-minted routes (internal/domains) are additional NAMES for |
| @@ -477,6 +494,14 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box { |
| } |
| routeVersions[r.Host] = r.Version |
| } |
| + if r.Kind != "static" && r.DBPath != "" && p.DBBytes != nil { |
| + if n := p.DBBytes(r.DBPath); n != nil { |
| + if routeDBBytes == nil { |
| + routeDBBytes = map[string]int64{} |
| + } |
| + routeDBBytes[r.Host] = *n |
| + } |
| + } |
| // Non-platform routes carry hostname readings too. A first-route |
| // claim (claims.go: "an empty route means this host is its own |
| // primary") materializes a ROUTE, not an alias row, so without this |
| @@ -590,7 +615,8 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box { |
| if p.MemTotalBytes != nil || p.MemAvailBytes != nil || p.Load1 != nil || |
| p.DiskFreeBytes != nil || p.Release != "" || p.Tunnel != "" || |
| p.InstanceID != "" || p.Platform != "" || p.SystemChannel != "" || p.System != nil || |
| - len(routeVersions) > 0 || len(aliasReadings) > 0 || len(p.Upstreams) > 0 { |
| + len(routeVersions) > 0 || len(aliasReadings) > 0 || len(p.Upstreams) > 0 || |
| + len(routeDBBytes) > 0 { |
| box.Metrics = &BoxMetrics{ |
| MemTotalBytes: p.MemTotalBytes, MemAvailBytes: p.MemAvailBytes, |
| Load1: p.Load1, DiskFreeBytes: p.DiskFreeBytes, |
| @@ -604,7 +630,8 @@ func CollectBox(routes []registry.Route, p Probes, now time.Time) Box { |
| // for a later probe tick to mutate under the marshaller. An empty |
| // map (rather than nil) reaching this line can only happen |
| // alongside another known field above, and omitempty drops it. |
| - Upstreams: p.Upstreams, |
| + Upstreams: p.Upstreams, |
| + RouteDBBytes: routeDBBytes, |
| } |
| } |
diff --git a/internal/status/status_test.go b/internal/status/status_test.go| index f85ece127..c16eb93d4 100644 |
| --- a/internal/status/status_test.go |
| +++ b/internal/status/status_test.go |
| @@ -1335,3 +1335,58 @@ func TestCollectBoxOmitsPlatformRoutesFromReadings(t *testing.T) { |
| t.Fatal("the custom domain must appear") |
| } |
| } |
| + |
| +// TestCollectBoxCarriesRouteDBBytes: every non-platform, non-static route |
| +// with a DBPath is sized through Probes.DBBytes into Metrics.RouteDBBytes. A |
| +// route the probe cannot size is absent rather than 0, and the hostname never |
| +// reaches the public fleet.json. |
| +func TestCollectBoxCarriesRouteDBBytes(t *testing.T) { |
| + routes := []registry.Route{ |
| + {Host: "sized.test", Kind: "instance", DBPath: "/data/sized.test.db"}, |
| + {Host: "missing.test", Kind: "instance", DBPath: "/data/missing.test.db"}, |
| + {Host: "nodb.test", Kind: "instance"}, |
| + {Host: "static.test", Kind: "static", DBPath: "/data/static.test.db"}, |
| + {Host: "platform.test", Kind: "instance", DBPath: "/data/sized.test.db", Platform: true}, |
| + } |
| + var asked []string |
| + probe := func(p string) *int64 { |
| + asked = append(asked, p) |
| + if p == "/data/sized.test.db" { |
| + n := int64(4096) |
| + return &n |
| + } |
| + return nil |
| + } |
| + b := status.CollectBox(routes, status.Probes{MachineID: "m", DBBytes: probe}, time.Now()) |
| + if b.Metrics == nil { |
| + t.Fatal("metrics block missing — RouteDBBytes should have attached it") |
| + } |
| + if len(b.Metrics.RouteDBBytes) != 1 || b.Metrics.RouteDBBytes["sized.test"] != 4096 { |
| + t.Fatalf("route_db_bytes = %v, want exactly {sized.test: 4096}", b.Metrics.RouteDBBytes) |
| + } |
| + if strings.Join(asked, ",") != "/data/sized.test.db,/data/missing.test.db" { |
| + t.Errorf("probed %v, want only the two instance routes with a DBPath", asked) |
| + } |
| + |
| + s := teststore(t) |
| + ctx := context.Background() |
| + if err := status.WriteBox(ctx, s, b); err != nil { |
| + t.Fatal(err) |
| + } |
| + f, err := status.BuildFleet(ctx, s, routes, time.Now(), 5*time.Minute) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + fj, err := json.Marshal(f) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if strings.Contains(string(fj), "sized.test") { |
| + t.Errorf("fleet.json leaked a route_db_bytes hostname: %s", fj) |
| + } |
| + |
| + // No probe at all: nothing sized, and no Metrics block from this alone. |
| + if b := status.CollectBox(routes[:1], status.Probes{MachineID: "m"}, time.Now()); b.Metrics != nil { |
| + t.Errorf("metrics = %+v with no DBBytes probe, want absent", b.Metrics) |
| + } |
| +} |
diff --git a/routes_cli.go b/routes_cli.go| index e6ed64469..4623fe8bf 100644 |
| --- a/routes_cli.go |
| +++ b/routes_cli.go |
| @@ -58,13 +58,13 @@ func runRoutes(args []string) { |
| } |
| shown := 0 |
| w := newTable(stdout) |
| - fmt.Fprintln(w, "HOST\tKIND\tUPSTREAM\tCHANNEL\tENVIRONMENT\tREPLICATION\tSTATUS\tHEALTH") |
| + fmt.Fprintln(w, "HOST\tKIND\tUPSTREAM\tCHANNEL\tENVIRONMENT\tREPLICATION\tSTATUS\tHEALTH\tDB") |
| for _, r := range rows { |
| if *match != "" && !strings.Contains(r.Host, *match) { |
| continue |
| } |
| shown++ |
| - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", |
| + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n", |
| r.Host, dashIfEmpty(r.Kind), dashIfEmpty(r.Upstream), dashIfEmpty(r.Channel), |
| // An empty ENVIRONMENT is a real answer — the route takes the |
| // app's default bundle — so it prints as a dash rather than a |
| @@ -79,7 +79,7 @@ func runRoutes(args []string) { |
| // where "no answer" cost one tenant 97% of a day's replica |
| // traffic. |
| replicationCell(r.Replication), |
| - dashIfEmpty(r.Status), dashIfEmpty(r.Health)) |
| + dashIfEmpty(r.Status), dashIfEmpty(r.Health), dbCell(r.DBBytes)) |
| // Path routes indented under the hostname whose traffic they split. |
| // A flipped prefix set has to be visible from the command operators |
| // already use, or a migration is invisible until something breaks. |
| @@ -98,7 +98,7 @@ func runRoutes(args []string) { |
| if p.Host != r.Host { |
| continue |
| } |
| - fmt.Fprintf(w, " /%s\tpath\t%s\t—\t—\t—\t—\t—\n", p.Prefix, dashIfEmpty(p.Target)) |
| + fmt.Fprintf(w, " /%s\tpath\t%s\t—\t—\t—\t—\t—\t—\n", p.Prefix, dashIfEmpty(p.Target)) |
| } |
| } |
| w.Flush() |
| @@ -131,3 +131,12 @@ func dashIfEmpty(s string) string { |
| } |
| return s |
| } |
| + |
| +// dbCell renders a route's database size, a dash (this table's own em dash) |
| +// when no box has measured it. |
| +func dbCell(n *int64) string { |
| + if n == nil { |
| + return "—" |
| + } |
| + return humanBytes(n) |
| +} |
diff --git a/store.go b/store.go| index 0db093234..cae6a08a1 100644 |
| --- a/store.go |
| +++ b/store.go |
| @@ -499,6 +499,21 @@ func printStoreRotate(w io.Writer, res apiStoreReply, finish bool) { |
| // "provisioned at" — review finding from the walk design: it MOVES on every |
| // rotate and every drift repair, so a fixed-sounding label would misdescribe |
| // it the very first time either happens. |
| +// printStoreSize prints the bucket's size from the console's daily storage |
| +// sweep, dated, because it is up to a day old: status reads the last sweep |
| +// rather than listing the bucket on every call. |
| +func printStoreSize(w io.Writer, sz *apiStoreSize) { |
| + switch { |
| + case sz == nil: |
| + mutedf(w, " size: not measured yet — the console sizes stores once a day") |
| + case sz.Unswept: |
| + warnf(w, "size: the %s sweep could not read this bucket (credential or endpoint) — no size known", sz.AsOf) |
| + default: |
| + n := sz.StoreBytes |
| + fmt.Fprintf(w, " size: %s (as of %s)\n", humanBytes(&n), sz.AsOf) |
| + } |
| +} |
| + |
| func printStoreStatus(w io.Writer, res apiStoreReply) { |
| c := colorFor(w) |
| fmt.Fprintf(w, "%s %s\n", c.head("store for"), res.App) |
| @@ -528,6 +543,7 @@ func printStoreStatus(w io.Writer, res apiStoreReply) { |
| p := res.Provisioned |
| fmt.Fprintf(w, " provisioned: bucket %s in %s, user %s, key %s — last credential change %s\n", |
| p.Bucket, p.Region, p.IAMUser, p.KeyID, p.At.Format(time.RFC3339)) |
| + printStoreSize(w, res.Size) |
| } else { |
| if res.Entitlement.Approved() { |
| mutedf(w, " nothing is provisioned yet. re-run carlos store create --app %s", res.App) |
diff --git a/store_test.go b/store_test.go| index 437596691..f2172ca07 100644 |
| --- a/store_test.go |
| +++ b/store_test.go |
| @@ -867,3 +867,29 @@ func TestStoreVerbDispatchAndRefusals(t *testing.T) { |
| t.Fatalf("bad app name = %q, want the name gate", out) |
| } |
| } |
| + |
| +// TestPrintStoreStatusSize: a provisioned store prints its swept size with |
| +// the sweep's date, "not measured yet" before any sweep, and a warning (never |
| +// a 0) when the sweep could not read the bucket. |
| +func TestPrintStoreStatusSize(t *testing.T) { |
| + base := apiStoreReply{App: "keymail", Backend: "aws", |
| + Declared: &apiStoreDeclared{EnvPrefix: "CARLOS_STORE"}, |
| + Provisioned: &apiStoreProvisioned{Bucket: "b1", Region: "eu-west-1", IAMUser: "u1", KeyID: "AKIA1"}} |
| + for _, tc := range []struct { |
| + name string |
| + size *apiStoreSize |
| + want string |
| + }{ |
| + {"swept", &apiStoreSize{AsOf: "2026-09-17", StoreBytes: 3 << 30}, "size: 3.0GB (as of 2026-09-17)"}, |
| + {"never", nil, "size: not measured yet"}, |
| + {"unswept", &apiStoreSize{AsOf: "2026-09-17", Unswept: true}, "could not read this bucket"}, |
| + } { |
| + var buf bytes.Buffer |
| + res := base |
| + res.Size = tc.size |
| + printStoreStatus(&buf, res) |
| + if !strings.Contains(buf.String(), tc.want) { |
| + t.Errorf("%s: output missing %q:\n%s", tc.name, tc.want, buf.String()) |
| + } |
| + } |
| +} |