amadan

carlos / platform Public

console: month-to-date account statements from metrics, edge and storage days

statementsPass folds the month's metrics day rollups, edge days, storage
days and census days into one usage.Statement per account, keyed the same
way census/storage already are: release.RouteAccount()'s sqid-shaped
output (e.g. release.OpsAccount == "bab"), not the console store's display
Name. unitOwner attributes a sampled unit to its route's account/app by
building each route's own expected unit names forward (activator.UnitName/
ExecName, mirroring OwnerAppMetrics) rather than parsing a host back out of
the sampled string; an unmatched unit falls to release.OpsAccount, never a
bare "ops" literal. The registry-open guard census pass already had is
extracted into openPassRegistry and shared by both passes.

Every account known to s.store.Accounts() (by Sqid) union'd with every
account touched by this month's usage gets a statement, including zero
usage — the accounts page must list every account. CoveredDays/DaysInMonth
are shared across every statement this pass writes: coverage is "did any
source object exist for this date", not per-account.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Paul Campbell aa54c466a8da980f495f1c7a63cf7f28cb17b6ce parent 9ccc83a
2 files changed, +392 −18
  • internal/console/economicspass.go +244 −18
  • internal/console/economicspass_test.go +148 −0
diff --git a/internal/console/economicspass.go b/internal/console/economicspass.go
index f6be211fc..f90e8d1cf 100644
--- a/internal/console/economicspass.go
+++ b/internal/console/economicspass.go
@@ -2,6 +2,7 @@ package console
import (
"context"
+ "encoding/json"
"fmt"
"log"
"os"
@@ -11,7 +12,10 @@ import (
"github.com/aws/aws-sdk-go-v2/aws"
awss3 "github.com/aws/aws-sdk-go-v2/service/s3"
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/metrics"
"github.com/carlosframework/platform/internal/objstore"
+ "github.com/carlosframework/platform/internal/registry"
"github.com/carlosframework/platform/internal/release"
"github.com/carlosframework/platform/internal/usage"
)
@@ -52,11 +56,31 @@ func (s *Server) economicsPass(ctx context.Context, now time.Time) error {
if err := s.storagePass(ctx, date); err != nil {
log.Printf("console: storage pass %s: %v", date, err)
}
- // Task 10 adds: s.statementsPass(ctx, now)
+ if err := s.statementsPass(ctx, now); err != nil {
+ log.Printf("console: statements pass: %v", err)
+ }
// Task 12 adds: s.billPass(ctx, now)
return nil
}
+// openPassRegistry opens the console's registry for a daily pass, refusing
+// early rather than letting registry.Open's sql.Open silently create and
+// migrate an EMPTY sqlite file at an unconfigured or unmounted path — a
+// pass that read that as "zero routes" would write a permanent, uncorrectable
+// all-zero day (create-only) and leave a stray sqlite file behind. Shared by
+// every pass that reads routes (censusPass, statementsPass); mirrors
+// instanceHealth's same-shaped guard for the empty-string case
+// (apiinstances.go:207).
+func (s *Server) openPassRegistry() (*registry.Registry, error) {
+ if s.cfg.Registry == "" {
+ return nil, fmt.Errorf("no registry configured")
+ }
+ if _, err := os.Stat(s.cfg.Registry); err != nil {
+ return nil, fmt.Errorf("registry %s: %w", s.cfg.Registry, err)
+ }
+ return openRegistry(s.cfg.Registry)
+}
+
// censusPass snapshots per-account app/route counts, create-only. Counted by
// account NAME: release.RouteAccount normalizes a route with no account
// (Route.Account == "") to release.OpsAccount, the deployment operator's
@@ -67,24 +91,9 @@ func (s *Server) censusPass(ctx context.Context, date string) error {
if _, _, err := s.cfg.Releases.Get(ctx, key); err == nil {
return nil // today already written (by us or a sibling console)
}
- // Guard both the unconfigured and the unmounted-volume case before
- // opening: registry.Open goes through sql.Open, which creates and
- // migrates an EMPTY sqlite file at whatever path it's given, missing
- // file or not. Left unguarded, a console booted with no registry path
- // or an unmounted registry volume would read that empty DB as zero
- // routes and CreateDayObject an all-zero census day — create-only, so
- // that day could never be corrected, plus a stray sqlite file left at
- // the registry path. Mirrors instanceHealth's same-shaped guard for the
- // empty-string case (apiinstances.go:207).
- if s.cfg.Registry == "" {
- return fmt.Errorf("console: census pass: no registry configured")
- }
- if _, err := os.Stat(s.cfg.Registry); err != nil {
- return fmt.Errorf("console: census pass: registry %s: %w", s.cfg.Registry, err)
- }
- reg, err := openRegistry(s.cfg.Registry)
+ reg, err := s.openPassRegistry()
if err != nil {
- return err
+ return fmt.Errorf("console: census pass: %w", err)
}
routes, err := reg.List()
if err != nil {
@@ -175,6 +184,223 @@ func (s *Server) storagePass(ctx context.Context, date string) error {
return err
}
+// unitOwner attributes a sampled systemd unit (or exec-child cgroup name) to
+// its route's account/app. Matching mirrors OwnerAppMetrics (metricsview.go
+// ~698-730): rather than parse a host back out of the unit string, it builds
+// each route's OWN expected unit names — activator.UnitName/ExecName for a
+// unit-backed or exec-backed instance route, plus the route's stamped Unit
+// column (#110's bespoke unit) — and compares those forward against the
+// sampled name. (Read first per the brief: neither UnitName nor ExecName
+// systemd-escapes the host — activator.UnitName is a bare string
+// concatenation and cgroup.ExecName likewise — so there is no unescaping
+// helper to reuse; building the expected name forward, the same direction
+// OwnerAppMetrics does it, is what keeps a hyphenated or dotted host correct
+// with no separate escaping step.) A unit matching no route — the console's
+// own unit, system units — attributes to release.OpsAccount, app "": platform
+// overhead.
+func unitOwner(routes []registry.Route, unit string) (acct, app string, ok bool) {
+ for _, rt := range routes {
+ if rt.Unit != "" && rt.Unit == unit {
+ return release.RouteAccount(rt), rt.App, true
+ }
+ if rt.Host != "" && (unit == activator.UnitName(rt.Host) || unit == activator.ExecName(rt.Host)) {
+ return release.RouteAccount(rt), rt.App, true
+ }
+ }
+ return release.OpsAccount, "", false
+}
+
+// statementsPass folds the month's metrics, edge and storage days (plus
+// census, for app_months) into one usage.Statement per account, create-or-
+// replace via usage.UpsertStatement — safe to run repeatedly as new days'
+// objects appear. Every UTC day of the month up to and including now is a
+// candidate; a day is COVERED (CoveredDays, shared across every account's
+// statement this pass writes, out of DaysInMonth) when at least one source
+// object — a metrics day rollup on any box, an edge day on any box, or the
+// day's storage sweep — exists for that date; a day with none of those
+// simply contributes nothing, uncovered, never a zero standing in for
+// missing data (package usage's "modelling, not invoice" posture).
+//
+// Known shortfall, recorded rather than fixed at modelling grade: an account
+// deleted mid-month drops out of s.store.Accounts() (its query filters
+// deleted_at), so this pass stops upserting its statement from the day of
+// deletion onward — the account's last statement object simply stops moving
+// forward, rather than the month closing it out explicitly.
+func (s *Server) statementsPass(ctx context.Context, now time.Time) error {
+ reg, err := s.openPassRegistry()
+ if err != nil {
+ return fmt.Errorf("console: statements pass: %w", err)
+ }
+ routes, err := reg.List()
+ if err != nil {
+ return fmt.Errorf("console: statements pass: %w", err)
+ }
+
+ nowUTC := now.UTC()
+ y, mo, today := nowUTC.Date()
+ daysInMonth := time.Date(y, mo+1, 0, 0, 0, 0, 0, time.UTC).Day()
+ month := usage.MonthOf(nowUTC)
+
+ metricsBoxes, err := s.cfg.Releases.ListDirs(ctx, metricsBoxesPrefix)
+ if err != nil {
+ return fmt.Errorf("console: statements pass: %w", err)
+ }
+ edgeBoxes, err := s.cfg.Releases.ListDirs(ctx, usage.EdgeBoxesPrefix)
+ if err != nil {
+ return fmt.Errorf("console: statements pass: %w", err)
+ }
+
+ meters := map[string]map[string]float64{} // account -> meter -> value
+ touch := func(acct string) map[string]float64 {
+ mm := meters[acct]
+ if mm == nil {
+ mm = map[string]float64{}
+ meters[acct] = mm
+ }
+ return mm
+ }
+
+ coveredDays := 0
+ for day := 1; day <= today; day++ {
+ date := time.Date(y, mo, day, 0, 0, 0, 0, time.UTC).Format("2006-01-02")
+ covered := false
+
+ for _, boxPrefix := range metricsBoxes {
+ box := strings.TrimSuffix(strings.TrimPrefix(boxPrefix, metricsBoxesPrefix), "/")
+ raw, _, gerr := s.cfg.Releases.Get(ctx, metrics.DayKey(box, date))
+ if gerr != nil {
+ continue // missing = skip, uncovered by this box
+ }
+ var roll metrics.DayRollup
+ if jerr := json.Unmarshal(raw, &roll); jerr != nil {
+ log.Printf("console: statements pass: parse %s: %v", metrics.DayKey(box, date), jerr)
+ continue
+ }
+ covered = true
+ for _, hour := range roll.Hours {
+ for unit, um := range hour.Units {
+ acct, _, _ := unitOwner(routes, unit)
+ mm := touch(acct)
+ var instN int
+ if cpu, ok := um["cpu"]; ok {
+ mm[usage.MeterCPUCoreHours] += cpu.Avg * float64(cpu.N) / 60
+ instN = cpu.N
+ }
+ if mem, ok := um["mem_bytes"]; ok {
+ mm[usage.MeterMemGBHours] += mem.Avg * float64(mem.N) / 60 / 1e9
+ if instN == 0 {
+ instN = mem.N
+ }
+ }
+ mm[usage.MeterInstanceHours] += float64(instN) / 60
+ }
+ }
+ }
+
+ for _, boxPrefix := range edgeBoxes {
+ box := strings.TrimSuffix(strings.TrimPrefix(boxPrefix, usage.EdgeBoxesPrefix), "/")
+ raw, _, gerr := s.cfg.Releases.Get(ctx, usage.EdgeDayKey(box, date))
+ if gerr != nil {
+ continue // missing = skip, uncovered by this box
+ }
+ var ed usage.EdgeDay
+ if jerr := json.Unmarshal(raw, &ed); jerr != nil {
+ log.Printf("console: statements pass: parse %s: %v", usage.EdgeDayKey(box, date), jerr)
+ continue
+ }
+ covered = true
+ for _, rows := range ed.Hours {
+ for _, row := range rows {
+ acct := row.Account
+ if acct == "" {
+ acct = release.OpsAccount
+ }
+ mm := touch(acct)
+ mm[usage.MeterEgressGB] += float64(row.BytesOut) / 1e9
+ mm[usage.MeterRequests1k] += float64(row.Requests) / 1000
+ }
+ }
+ }
+
+ if raw, _, gerr := s.cfg.Releases.Get(ctx, usage.StorageDayKey(date)); gerr == nil {
+ var sd usage.StorageDay
+ if jerr := json.Unmarshal(raw, &sd); jerr != nil {
+ log.Printf("console: statements pass: parse %s: %v", usage.StorageDayKey(date), jerr)
+ } else {
+ covered = true
+ for key, ap := range sd.Apps {
+ acct := key
+ if i := strings.IndexByte(key, '/'); i >= 0 {
+ acct = key[:i]
+ }
+ bytesTotal := ap.BucketBytes + ap.StoreBytes
+ touch(acct)[usage.MeterStorageGBMonth] += (float64(bytesTotal) / 1e9) / float64(daysInMonth)
+ }
+ }
+ } // missing = skip, uncovered
+
+ // Census does not gate coverage (the brief's covered-day sources are
+ // metrics/edge/storage only) but still feeds app_months when present.
+ if raw, _, gerr := s.cfg.Releases.Get(ctx, usage.CensusDayKey(date)); gerr == nil {
+ var cd usage.CensusDay
+ if jerr := json.Unmarshal(raw, &cd); jerr != nil {
+ log.Printf("console: statements pass: parse %s: %v", usage.CensusDayKey(date), jerr)
+ } else {
+ for acct, ca := range cd.Accounts {
+ touch(acct)[usage.MeterAppMonths] += float64(ca.Apps) / float64(daysInMonth)
+ }
+ }
+ }
+
+ if covered {
+ coveredDays++
+ }
+ }
+
+ // Every account gets a statement this pass, even one with zero usage —
+ // the accounts page must list it (brief step 3). The union is the
+ // accumulated meter keys (already RouteAccount()-shaped, e.g.
+ // release.OpsAccount == "bab" — see censusPass/storagePass, same key
+ // space) plus every live console account's Sqid. Deliberately NOT the
+ // console account's display Name: Name ("ops") and RouteAccount's sqid
+ // output ("bab") are different strings for the SAME built-in account
+ // (Task 8's census golden pins "bab" as the key), so a name-keyed union
+ // would never actually join a zero-usage account into the sqid-keyed
+ // universe census/edge/storage already write under — see task-10-report
+ // for the full reasoning.
+ accounts, err := s.store.Accounts()
+ if err != nil {
+ return fmt.Errorf("console: statements pass: %w", err)
+ }
+ universe := map[string]bool{}
+ for acct := range meters {
+ universe[acct] = true
+ }
+ for _, a := range accounts {
+ universe[a.Sqid] = true
+ }
+
+ for acct := range universe {
+ mm := meters[acct]
+ if mm == nil {
+ mm = map[string]float64{}
+ }
+ st := usage.Statement{
+ V: 1,
+ Account: acct,
+ Month: month,
+ Meters: mm,
+ CoveredDays: coveredDays,
+ DaysInMonth: daysInMonth,
+ UpdatedAt: nowUTC,
+ }
+ if err := usage.UpsertStatement(ctx, s.cfg.Releases, st); err != nil {
+ return fmt.Errorf("console: statements pass: upsert %s: %w", acct, err)
+ }
+ }
+ return nil
+}
+
// sizeStore routes through the test seam or the real S3 sizer.
func (s *Server) sizeStore(ctx context.Context, p *objstore.Provisioned) (int64, error) {
if s.storeSizer != nil {
diff --git a/internal/console/economicspass_test.go b/internal/console/economicspass_test.go
index c89e34cf7..d81a8d128 100644
--- a/internal/console/economicspass_test.go
+++ b/internal/console/economicspass_test.go
@@ -4,16 +4,34 @@ import (
"context"
"encoding/json"
"errors"
+ "math"
"os"
"path/filepath"
"testing"
"time"
+ "github.com/carlosframework/platform/internal/metrics"
"github.com/carlosframework/platform/internal/objstore"
+ "github.com/carlosframework/platform/internal/registry"
"github.com/carlosframework/platform/internal/release"
"github.com/carlosframework/platform/internal/usage"
)
+// putStoreJSON marshals v and writes it straight to the fixture bucket at
+// key, create-only. Named distinctly from apiconfig_test.go's putJSON (an
+// HTTP PUT helper against the running server) — same package, so a same-
+// named bucket-level helper would collide with it.
+func putStoreJSON(t *testing.T, s release.Store, key string, v any) {
+ t.Helper()
+ raw, err := json.Marshal(v)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := s.Put(context.Background(), key, raw, ""); err != nil {
+ t.Fatal(err)
+ }
+}
+
// fixtureAcct/fixtureApp are the account and app buildFixtures (view_test.go)
// actually ships releases under: release.OpsAccount ("bab") owns both "kass"
// and "otherapp" routes, but only "kass" gets shipped artifacts
@@ -256,3 +274,133 @@ func TestStoragePassDegradesOnStoreRecordReadError(t *testing.T) {
t.Fatalf("app.StoreBytes = %d, want 0 when unswept", app.StoreBytes)
}
}
+
+// TestStatementsPassGolden pins the canonical meter derivations end to end
+// off bucket objects alone — no live boxes needed. cpu avg 0.5 for 60
+// samples (60s apart) in one hour is 0.5 core-hours; mem avg 1e9 bytes over
+// the same 60 samples is 1 GB-hour and, from the same sample count,
+// 1 instance-hour. The route is unit-backed (Host=a.example.com, no
+// Route.Unit), so unitOwner must match it via activator.UnitName, exactly
+// the forward-construction OwnerAppMetrics uses.
+func TestStatementsPassGolden(t *testing.T) {
+ fx := newConsoleFixture(t)
+ ctx := context.Background()
+ now := time.Date(2026, 8, 11, 6, 0, 0, 0, time.UTC)
+
+ reg, err := registry.Open(fx.regPath)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.Upsert(registry.Route{Host: "a.example.com", Kind: "instance",
+ Status: "ready", App: fixtureApp, Version: "v1"}); err != nil {
+ t.Fatal(err)
+ }
+
+ // One box, one day of metrics: unit carlos-app@a.example.com with
+ // cpu avg 0.5 for 60 samples in hour 09 (= 0.5 core-hours), mem avg
+ // 1e9 bytes (= 1 GB-hour, 1 instance-hour).
+ cpu, mem := 0.5, 1e9
+ roll := metrics.DayRollup{V: 1, Box: "box1", Date: "2026-08-10", Hours: map[string]metrics.HourAgg{
+ "09": {Units: map[string]map[string]metrics.Agg{
+ "carlos-app@a.example.com.service": {
+ "cpu": {Avg: cpu, N: 60},
+ "mem_bytes": {Avg: mem, N: 60},
+ },
+ }},
+ }}
+ putStoreJSON(t, fx.rel, metrics.DayKey("box1", "2026-08-10"), roll)
+
+ // Edge day: 2 GB out, 4000 requests for the same route's account.
+ putStoreJSON(t, fx.rel, usage.EdgeDayKey("box1", "2026-08-10"), usage.EdgeDay{
+ V: 1, Box: "box1", Date: "2026-08-10",
+ Hours: map[string]map[string]usage.EdgeRow{"09": {
+ "a.example.com": {Account: fixtureAcct, App: fixtureApp, Requests: 4000, BytesOut: 2e9},
+ }},
+ })
+
+ if err := fx.srv.statementsPass(ctx, now); err != nil {
+ t.Fatal(err)
+ }
+
+ raw, _, err := fx.rel.Get(ctx, usage.StatementKey(fixtureAcct, "2026-08"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var st usage.Statement
+ if err := json.Unmarshal(raw, &st); err != nil {
+ t.Fatal(err)
+ }
+ approxF := func(name string, got, want float64) {
+ t.Helper()
+ if math.Abs(got-want) > 1e-9 {
+ t.Fatalf("%s = %v, want %v", name, got, want)
+ }
+ }
+ approxF("cpu", st.Meters[usage.MeterCPUCoreHours], 0.5)
+ approxF("mem", st.Meters[usage.MeterMemGBHours], 1.0)
+ approxF("instance_hours", st.Meters[usage.MeterInstanceHours], 1.0)
+ approxF("egress", st.Meters[usage.MeterEgressGB], 2.0)
+ approxF("requests", st.Meters[usage.MeterRequests1k], 4.0)
+ if st.DaysInMonth != 31 || st.CoveredDays < 1 {
+ t.Fatalf("coverage = %d/%d", st.CoveredDays, st.DaysInMonth)
+ }
+}
+
+// TestStatementsPassZeroUsageAccountListed: an account with a live route but
+// no metrics/edge/storage data this month still gets a statement — the
+// accounts page must be able to list every account, not just ones with
+// usage — and unitOwner falls a completely unmatched unit through to
+// release.OpsAccount rather than a bare "ops" literal.
+func TestStatementsPassZeroUsageAccountListed(t *testing.T) {
+ fx := newConsoleFixture(t)
+ ctx := context.Background()
+ now := time.Date(2026, 8, 11, 6, 0, 0, 0, time.UTC)
+
+ if err := fx.srv.statementsPass(ctx, now); err != nil {
+ t.Fatal(err)
+ }
+ raw, _, err := fx.rel.Get(ctx, usage.StatementKey(release.OpsAccount, "2026-08"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ var st usage.Statement
+ if err := json.Unmarshal(raw, &st); err != nil {
+ t.Fatal(err)
+ }
+ if st.Account != release.OpsAccount {
+ t.Fatalf("st.Account = %q, want %q", st.Account, release.OpsAccount)
+ }
+ for meter, v := range st.Meters {
+ if v != 0 {
+ t.Fatalf("zero-usage statement has non-zero %s = %v", meter, v)
+ }
+ }
+}
+
+// TestUnitOwnerUnmatchedFallsToOpsAccount pins the correction over the
+// brief's literal text: the fallback is release.OpsAccount ("bab"), never
+// the bare string "ops".
+func TestUnitOwnerUnmatchedFallsToOpsAccount(t *testing.T) {
+ acct, app, ok := unitOwner(nil, "console.service")
+ if ok {
+ t.Fatalf("unitOwner matched an empty route list: acct=%q app=%q", acct, app)
+ }
+ if acct != release.OpsAccount {
+ t.Fatalf("unmatched unit acct = %q, want release.OpsAccount (%q)", acct, release.OpsAccount)
+ }
+ if app != "" {
+ t.Fatalf("unmatched unit app = %q, want empty", app)
+ }
+}
+
+// TestUnitOwnerMatchesStampedUnitColumn: a route with a non-empty Unit
+// column matches that exact unit name, independent of Host.
+func TestUnitOwnerMatchesStampedUnitColumn(t *testing.T) {
+ routes := []registry.Route{
+ {Host: "bespoke.example.com", App: "bespoke", Unit: "carlos-bespoke.service"},
+ }
+ acct, app, ok := unitOwner(routes, "carlos-bespoke.service")
+ if !ok || app != "bespoke" || acct != release.OpsAccount {
+ t.Fatalf("unitOwner(stamped unit) = %q %q %v", acct, app, ok)
+ }
+}