amadan
Fix round four in the statement walk and the partial-month sums; count durability days per account
Statement catch-up (Astra R3, Opus N4). A month is absent only when no account has a statement for it: a missing platform statement in a month that has a tenant's is an open month, where it used to put the oldest metered month outside the range for good. A statement that will not parse is logged once per process and no longer sweeps its month every pass, which a sweep could never repair. A store error concludes nothing for the month this pass. Months seen with no statement are remembered until this process sweeps them. Partial-month sums (Astra R5, Opus N2). Days are summed in date order, so the same statement gives the same answer on every run, including at the edge of the tolerance. A statement whose days contradict it is a different error from one with no days at all, and the invoice pass logs it by name. Durability fold. The span check uses a five-minute tolerance, where comparing clock hours still held a new box's first day when a race of seconds crossed the top of an hour (Opus N6). Durability days are now counted per account: a box-day attested with an exclusion set is complete for every account the box did not exclude, and each statement's source days are reduced for its own account only. This is the console half of the scoped attestation chosen in the attestation plan. No box attests, so it changes nothing in production. Tests. The partial-month boundary is pinned case by case (midnight edges, a nanosecond either side, a paid-through at noon, a quantity change at noon, a lapse inside a day, a span inside one day). The day-sum test feeds all five billed meters. A full month's frozen bytes carry none of the partial-month fields. The production preparation gate is shown to apply to a partial month. The classifier is pinned against every S3 transfer usage type the flagship account wrote from July to September 2026, read from Cost Explorer on 2026-10-01: Ireland is written EU-, which no fixture used.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
9 files changed,
+691
−106
internal/awsbilling/classify_test.go+41 −0internal/console/billingcalculation.go+36 −14internal/console/billingcalculation_test.go+215 −4internal/console/billinginvoice.go+1 −1internal/console/durabilityfold.go+45 −16internal/console/economicspass.go+118 −58internal/console/meterfolds_test.go+223 −10internal/console/server.go+4 −0internal/usage/statement.go+8 −3
diff --git a/internal/awsbilling/classify_test.go b/internal/awsbilling/classify_test.go| index 75f510af0..2f7ff9b67 100644 |
| --- a/internal/awsbilling/classify_test.go |
| +++ b/internal/awsbilling/classify_test.go |
| @@ -167,6 +167,47 @@ func TestChargedS3TransferTable(t *testing.T) { |
| } |
| } |
| +// Every S3 transfer usage type Cost Explorer reported for the flagship |
| +// account (271376211898) from July to September 2026, read on 2026-10-01. |
| +// Ireland appears under the legacy "EU-" prefix and us-east-1 with no |
| +// prefix at all; the fixtures elsewhere in this file use "EUW1-", which |
| +// that account never writes. Each real name must classify the way the |
| +// ruling needs: outbound and inter-region outbound count when charged, |
| +// inbound never counts, and none of them falls through to "not transfer", |
| +// which would let a charged byte vanish without a hold. |
| +func TestRealFlagshipUsageTypesClassify(t *testing.T) { |
| + out := []string{ |
| + "DataTransfer-Out-Bytes", "EU-DataTransfer-Out-Bytes", "EUW2-DataTransfer-Out-Bytes", "APS2-DataTransfer-Out-Bytes", |
| + "EU-APS2-AWS-Out-Bytes", "EU-EUW2-AWS-Out-Bytes", "EU-USE1-AWS-Out-Bytes", |
| + "APS2-EU-AWS-Out-Bytes", "EUW2-EU-AWS-Out-Bytes", "EUW2-USE1-AWS-Out-Bytes", |
| + "USE1-EU-AWS-Out-Bytes", "USE1-EUW2-AWS-Out-Bytes", |
| + } |
| + in := []string{ |
| + "DataTransfer-In-Bytes", "EU-DataTransfer-In-Bytes", "EUW2-DataTransfer-In-Bytes", "APS2-DataTransfer-In-Bytes", |
| + "EU-APS2-AWS-In-Bytes", "EU-EUW2-AWS-In-Bytes", "EU-USE1-AWS-In-Bytes", |
| + "EUW2-USE1-AWS-In-Bytes", "USE1-EU-AWS-In-Bytes", "USE1-EUW2-AWS-In-Bytes", |
| + } |
| + for _, ut := range out { |
| + bucket, n, charged, reason := ChargedS3Transfer(s3Line(t, ut, "2", "0.04")) |
| + if !charged || n != 2_000_000_000 || reason != ReasonCharged || bucket == "" { |
| + t.Errorf("%s charged: got charged=%v bytes=%d reason=%q", ut, charged, n, reason) |
| + } |
| + if _, _, charged, reason := ChargedS3Transfer(s3Line(t, ut, "2", "0")); charged || reason != ReasonFree { |
| + t.Errorf("%s at zero cost: charged=%v reason=%q", ut, charged, reason) |
| + } |
| + } |
| + for _, ut := range in { |
| + if _, _, charged, reason := ChargedS3Transfer(s3Line(t, ut, "2", "0")); charged || reason != ReasonInbound { |
| + t.Errorf("%s: charged=%v reason=%q, want inbound and uncounted", ut, charged, reason) |
| + } |
| + // AWS does not charge inbound transfer. If it ever does, nobody has |
| + // been shown to owe it: hold. |
| + if _, _, charged, reason := ChargedS3Transfer(s3Line(t, ut, "2", "0.04")); charged || !IsHold(reason) { |
| + t.Errorf("%s with a cost: charged=%v reason=%q, want a hold", ut, charged, reason) |
| + } |
| + } |
| +} |
| + |
| func TestIsHold(t *testing.T) { |
| for _, r := range []string{ReasonUnknownTransfer, ReasonUnexpectedCharge, ReasonTransferAdjustment, ReasonNoResource, ReasonNotHourly, ReasonBadAmount} { |
| if !IsHold(r) { |
diff --git a/internal/console/billingcalculation.go b/internal/console/billingcalculation.go| index e5f8e59ae..d19df5356 100644 |
| --- a/internal/console/billingcalculation.go |
| +++ b/internal/console/billingcalculation.go |
| @@ -8,6 +8,7 @@ import ( |
| "errors" |
| "math" |
| "math/big" |
| + "sort" |
| "time" |
| "github.com/carlosframework/platform/internal/plans" |
| @@ -124,23 +125,35 @@ func priceBillingCalculation(statement usage.Statement, history []plans.Assignme |
| } |
| // errBillingDays means a partly covered month cannot be billed from this |
| -// statement: it carries no daily meters, or they do not add up to its |
| -// monthly totals. The month holds; it is never billed on the monthly total, |
| -// which would charge usage from outside the paid days. |
| +// statement because it has no daily meters at all: it was written before |
| +// statements carried them. Billing only the paid days needs the days, so |
| +// the month holds. It is quiet: nothing is wrong with the statement. |
| var errBillingDays = errors.New("billing_statement_days_unusable") |
| +// errBillingDaysMismatch means the statement's daily meters contradict it: |
| +// a day outside the month, or days that do not sum to the monthly total of |
| +// a billed meter. That is a fold defect, not a missing input, and every |
| +// partly covered month it touches would hold for good, so it is never |
| +// quiet: the invoice pass logs it by name on every attempt. |
| +var errBillingDaysMismatch = errors.New("billing_statement_days_mismatch") |
| + |
| // billingUsageOnPaidDays returns each billed meter summed over the UTC days |
| -// of the month that lie WHOLLY inside one paid interval, and those days. |
| -// The day a plan starts or ends on is not counted: its usage cannot be |
| -// split, and the rounding goes to the customer. |
| +// that lie wholly inside one paid span, and those days in order. |
| +// |
| +// A day counts only when all 24 hours are inside a single covered interval: |
| +// the day a plan starts and the day it ends are not counted, because the |
| +// statement cannot say which of that day's usage fell on which side. That |
| +// rounds toward the customer by at most the boundary days. |
| // |
| -// The statement's days must account for its monthly totals, meter by |
| -// meter. A statement written before daily meters existed, or one whose days |
| -// disagree with its totals, is refused. |
| +// Before anything is counted the days must account for the statement: every |
| +// day inside the month, and each billed meter's days summing to its monthly |
| +// total. The days are walked in date order, so the sums, and with them |
| +// whether a statement at the edge of the tolerance is accepted, are the |
| +// same on every run. |
| func billingUsageOnPaidDays(statement usage.Statement, covered []plans.CoveredInterval) (map[string]float64, []string, error) { |
| start, err := time.Parse("2006-01", statement.Month) |
| if err != nil { |
| - return nil, nil, errBillingDays |
| + return nil, nil, errBillingDaysMismatch |
| } |
| end := start.AddDate(0, 1, 0) |
| if statement.Days == nil { |
| @@ -152,13 +165,22 @@ func billingUsageOnPaidDays(statement usage.Statement, covered []plans.CoveredIn |
| } |
| } |
| } |
| + dates := make([]string, 0, len(statement.Days)) |
| + for date := range statement.Days { |
| + dates = append(dates, date) |
| + } |
| + sort.Strings(dates) |
| sums := map[string]float64{} |
| - for date, mm := range statement.Days { |
| + for _, date := range dates { |
| day, perr := time.Parse("2006-01-02", date) |
| if perr != nil || day.Before(start) || !day.Before(end) { |
| - return nil, nil, errBillingDays |
| + return nil, nil, errBillingDaysMismatch |
| } |
| - for meter, v := range mm { |
| + for _, meter := range monthlyBillingMeters { |
| + v, ok := statement.Days[date][meter] |
| + if !ok { |
| + continue |
| + } |
| if v < 0 || math.IsNaN(v) || math.IsInf(v, 0) { |
| return nil, nil, errBillingMeter |
| } |
| @@ -168,7 +190,7 @@ func billingUsageOnPaidDays(statement usage.Statement, covered []plans.CoveredIn |
| for _, meter := range monthlyBillingMeters { |
| total := statement.Meters[meter] |
| if diff := math.Abs(sums[meter] - total); diff > 1e-9*math.Max(1, math.Abs(total)) { |
| - return nil, nil, errBillingDays |
| + return nil, nil, errBillingDaysMismatch |
| } |
| } |
| out := map[string]float64{} |
diff --git a/internal/console/billingcalculation_test.go b/internal/console/billingcalculation_test.go| index 590ac1edd..5b614f3b9 100644 |
| --- a/internal/console/billingcalculation_test.go |
| +++ b/internal/console/billingcalculation_test.go |
| @@ -6,6 +6,7 @@ import ( |
| "errors" |
| "math" |
| "reflect" |
| + "strings" |
| "testing" |
| "time" |
| @@ -207,24 +208,139 @@ func TestBillingCalculationNeverCountsABoundaryDay(t *testing.T) { |
| } |
| } |
| +// The boundary of the whole-paid-day rule, case by case. A UTC day is |
| +// billed only when every instant of it lies inside one paid span. In each |
| +// case the usage billed is the counted days' and nothing else, and the |
| +// counted days never cover more time than the plan was paid for, so the |
| +// allowance (weighted over the paid time) is never smaller than the |
| +// allowance those days alone would earn: every boundary rounds toward the |
| +// customer. |
| +func TestBillingCalculationPaidDayBoundaries(t *testing.T) { |
| + at := func(day, hour, min int, nsec int) time.Time { |
| + return time.Date(2026, 9, day, hour, min, 0, nsec, time.UTC) |
| + } |
| + base := func() plans.Assignment { _, h := calculationFixture(plans.PlanBusiness, 10); return h[0] } |
| + with := func(a plans.Assignment, rev int64, effective time.Time, mut func(*plans.Assignment)) plans.Assignment { |
| + a.Revision, a.EffectiveAt = rev, effective |
| + if mut != nil { |
| + mut(&a) |
| + } |
| + return a |
| + } |
| + status := func(st string) func(*plans.Assignment) { return func(a *plans.Assignment) { a.Status = st } } |
| + cases := []struct { |
| + name string |
| + history []plans.Assignment |
| + first, last string // counted days; empty first means none |
| + count int |
| + }{ |
| + {"starts at midnight exactly: that day counts", |
| + []plans.Assignment{with(base(), 1, at(10, 0, 0, 0), nil)}, "2026-09-10", "2026-09-30", 21}, |
| + {"starts one nanosecond after midnight: that day does not", |
| + []plans.Assignment{with(base(), 1, at(10, 0, 0, 1), nil)}, "2026-09-11", "2026-09-30", 20}, |
| + {"starts in the last minute of a day: that day does not", |
| + []plans.Assignment{with(base(), 1, at(10, 23, 59, 0), nil)}, "2026-09-11", "2026-09-30", 20}, |
| + {"ends at midnight exactly: the day before counts, that day does not", |
| + []plans.Assignment{base(), with(base(), 2, at(20, 0, 0, 0), status("canceled"))}, "2026-09-01", "2026-09-19", 19}, |
| + {"ends one nanosecond before midnight: its day does not count", |
| + []plans.Assignment{base(), with(base(), 2, at(19, 23, 59, 59_999_999_999), status("canceled"))}, "2026-09-01", "2026-09-18", 18}, |
| + {"paid through noon on the twentieth: the twentieth does not count", |
| + []plans.Assignment{with(base(), 1, at(1, 0, 0, 0), func(a *plans.Assignment) { a.PaidThrough = at(20, 12, 0, 0) })}, "2026-09-01", "2026-09-19", 19}, |
| + {"a quantity change at noon joins its spans: every day counts", |
| + []plans.Assignment{with(base(), 1, at(10, 0, 0, 0), nil), with(base(), 2, at(15, 12, 0, 0), func(a *plans.Assignment) { a.Quantity = 12 })}, "2026-09-10", "2026-09-30", 21}, |
| + {"starts on the last day at midnight: one day", |
| + []plans.Assignment{with(base(), 1, at(30, 0, 0, 0), nil)}, "2026-09-30", "2026-09-30", 1}, |
| + {"starts during the last day: no day, nothing billed", |
| + []plans.Assignment{with(base(), 1, at(30, 9, 0, 0), nil)}, "", "", 0}, |
| + {"paid for six hours inside one day: no day, nothing billed", |
| + []plans.Assignment{with(base(), 1, at(12, 6, 0, 0), nil), with(base(), 2, at(12, 12, 0, 0), status("canceled"))}, "", "", 0}, |
| + } |
| + for _, c := range cases { |
| + t.Run(c.name, func(t *testing.T) { |
| + st, _ := calculationFixture(plans.PlanBusiness, 10) |
| + st.Days, st.Meters[usage.MeterInstanceHours] = septemberDays(usage.MeterInstanceHours, 100) |
| + got, err := priceBillingCalculation(st, c.history) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if !got.Allowance.Partial { |
| + t.Fatal("not priced as a partial month") |
| + } |
| + if len(got.CountedDays) != c.count || (c.count > 0 && (got.CountedDays[0] != c.first || got.CountedDays[c.count-1] != c.last)) { |
| + t.Fatalf("counted days %v, want %d from %s to %s", got.CountedDays, c.count, c.first, c.last) |
| + } |
| + if want := int64(c.count) * 100 * 3600; got.Usage.AwakeSeconds != want { |
| + t.Fatalf("billed %d awake seconds, want the counted days' %d", got.Usage.AwakeSeconds, want) |
| + } |
| + var paid time.Duration |
| + for _, iv := range got.Allowance.Covered { |
| + paid += iv.To.Sub(iv.From) |
| + } |
| + if counted := time.Duration(c.count) * 24 * time.Hour; counted > paid { |
| + t.Fatalf("counted %s of usage against %s of paid time", counted, paid) |
| + } |
| + if c.count == 0 && got.Invoice.TotalCents != 0 { |
| + t.Fatalf("billed %d cents with no whole paid day", got.Invoice.TotalCents) |
| + } |
| + }) |
| + } |
| +} |
| + |
| +// A lapse inside a day takes that day out: an hour unpaid on the |
| +// fifteenth means the fifteenth is not billed, and the days either side |
| +// are. |
| +func TestBillingCalculationALapseInsideADayTakesTheDayOut(t *testing.T) { |
| + st, h := calculationFixture(plans.PlanBusiness, 10) |
| + lapse, back := h[0], h[0] |
| + lapse.Revision, lapse.Status, lapse.EffectiveAt = 2, "past_due", time.Date(2026, 9, 15, 9, 0, 0, 0, time.UTC) |
| + back.Revision, back.EffectiveAt = 3, time.Date(2026, 9, 15, 10, 0, 0, 0, time.UTC) |
| + st.Days, st.Meters[usage.MeterInstanceHours] = septemberDays(usage.MeterInstanceHours, 100) |
| + got, err := priceBillingCalculation(st, append(h, lapse, back)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(got.CountedDays) != 29 { |
| + t.Fatalf("counted %d days, want 29: %v", len(got.CountedDays), got.CountedDays) |
| + } |
| + for _, d := range got.CountedDays { |
| + if d == "2026-09-15" { |
| + t.Fatal("the day of the lapse was billed") |
| + } |
| + } |
| + if len(got.Allowance.Covered) != 2 { |
| + t.Fatalf("covered %+v, want two spans around the lapse", got.Allowance.Covered) |
| + } |
| +} |
| + |
| // Daily meters that do not account for the monthly total, or name a day |
| -// outside the month, hold a partly covered month. |
| +// outside the month, hold a partly covered month, and loudly: the |
| +// statement contradicts itself. A statement with no daily meters at all |
| +// (written before they existed) holds quietly. |
| func TestBillingCalculationHoldsAPartialMonthOnUnusableDays(t *testing.T) { |
| mid := time.Date(2026, 9, 16, 0, 0, 0, 0, time.UTC) |
| st, h := calculationFixture(plans.PlanBusiness, 10) |
| h[0].EffectiveAt = mid |
| st.Days, st.Meters[usage.MeterInstanceHours] = septemberDays(usage.MeterInstanceHours, 100) |
| st.Meters[usage.MeterInstanceHours] += 50 // the days no longer add up |
| - if _, err := priceBillingCalculation(st, h); !errors.Is(err, errBillingDays) { |
| - t.Fatalf("days that do not sum to the total: %v", err) |
| + _, err := priceBillingCalculation(st, h) |
| + if !errors.Is(err, errBillingDaysMismatch) || billingInvoiceQuiet(err) || billingInvoiceDiagnostic(err) != "billing_statement_days_mismatch" { |
| + t.Fatalf("days that do not sum to the total: %v (quiet=%v, diagnostic %q)", err, billingInvoiceQuiet(err), billingInvoiceDiagnostic(err)) |
| } |
| st, h = calculationFixture(plans.PlanBusiness, 10) |
| h[0].EffectiveAt = mid |
| st.Days = map[string]map[string]float64{"2026-10-01": {usage.MeterInstanceHours: 1}} |
| st.Meters[usage.MeterInstanceHours] = 1 |
| - if _, err := priceBillingCalculation(st, h); !errors.Is(err, errBillingDays) { |
| + if _, err := priceBillingCalculation(st, h); !errors.Is(err, errBillingDaysMismatch) { |
| t.Fatalf("a day outside the month: %v", err) |
| } |
| + // Written before statements carried days: usage, and no days at all. |
| + st, h = calculationFixture(plans.PlanBusiness, 10) |
| + h[0].EffectiveAt = mid |
| + st.Meters[usage.MeterInstanceHours] = 3000 |
| + _, err = priceBillingCalculation(st, h) |
| + if !errors.Is(err, errBillingDays) || errors.Is(err, errBillingDaysMismatch) || !billingInvoiceQuiet(err) { |
| + t.Fatalf("a statement with no daily meters: %v (quiet=%v)", err, billingInvoiceQuiet(err)) |
| + } |
| // A partly covered month that recorded nothing bills nothing. |
| st, h = calculationFixture(plans.PlanBusiness, 10) |
| h[0].EffectiveAt = mid |
| @@ -234,6 +350,85 @@ func TestBillingCalculationHoldsAPartialMonthOnUnusableDays(t *testing.T) { |
| } |
| } |
| +// The same statement and history give the same answer on every run, down |
| +// to the bytes, including at the edge of the day-sum tolerance. The days |
| +// are summed in date order: summed in map order, these three values come |
| +// to 1000.3000000000001 or to 1000.3 depending on the order, and against |
| +// this total one of those is inside the tolerance and the other outside. |
| +func TestBillingCalculationPartialMonthIsDeterministic(t *testing.T) { |
| + build := func() (usage.Statement, []plans.Assignment) { |
| + st, h := calculationFixture(plans.PlanPersonal, 1) |
| + h[0].EffectiveAt = time.Date(2026, 9, 10, 0, 0, 0, 0, time.UTC) |
| + st.Days = map[string]map[string]float64{ |
| + "2026-09-12": {usage.MeterEgressGB: 1000}, |
| + "2026-09-13": {usage.MeterEgressGB: 0.1}, |
| + "2026-09-14": {usage.MeterEgressGB: 0.2}, |
| + } |
| + st.Meters[usage.MeterEgressGB] = 1000.2999989997 |
| + return st, h |
| + } |
| + var first []byte |
| + var firstErr error |
| + for i := 0; i < 300; i++ { |
| + st, h := build() |
| + got, err := priceBillingCalculation(st, h) |
| + var raw []byte |
| + if err == nil { |
| + raw, _ = json.Marshal(got) |
| + } |
| + if i == 0 { |
| + first, firstErr = raw, err |
| + continue |
| + } |
| + if (err == nil) != (firstErr == nil) || string(raw) != string(first) { |
| + t.Fatalf("run %d differs from run 0: err %v vs %v", i, err, firstErr) |
| + } |
| + } |
| + // And a partly covered month's frozen bytes are stable when it prices. |
| + st, h := calculationFixture(plans.PlanBusiness, 10) |
| + h[0].EffectiveAt = time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) |
| + st.Days, st.Meters[usage.MeterInstanceHours] = septemberDays(usage.MeterInstanceHours, 100) |
| + a, err := priceBillingCalculation(st, h) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + b, _ := priceBillingCalculation(st, h) |
| + ra, _ := json.Marshal(a) |
| + rb, _ := json.Marshal(b) |
| + if string(ra) != string(rb) || !strings.Contains(string(ra), `"counted_days"`) { |
| + t.Fatal("a partial month's frozen calculation is not stable, or does not record its counted days") |
| + } |
| +} |
| + |
| +// A month covered end to end freezes exactly as it did before partial |
| +// months existed: the old arithmetic name, and none of the partial-month |
| +// fields, so a calculation frozen by an older build compares equal. |
| +func TestBillingCalculationFullMonthBytesAreUnchanged(t *testing.T) { |
| + st, h := calculationFixture(plans.PlanPersonal, 1) |
| + st.Meters[usage.MeterEgressGB] = 10 |
| + got, err := priceBillingCalculation(st, h) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + raw, _ := json.Marshal(got) |
| + for _, absent := range []string{`"counted_days"`, `"partial"`, `"covered"`, `"days"`} { |
| + if strings.Contains(string(raw), absent) { |
| + t.Fatalf("a full month's frozen calculation carries %s: %s", absent, raw) |
| + } |
| + } |
| + if got.Arithmetic != "base_units_floor_v1" { |
| + t.Fatalf("arithmetic %q", got.Arithmetic) |
| + } |
| + // With daily meters on the statement the calculation is the same; only |
| + // the embedded statement gains its days. |
| + st2, h2 := calculationFixture(plans.PlanPersonal, 1) |
| + st2.Days, st2.Meters[usage.MeterEgressGB] = map[string]map[string]float64{"2026-09-05": {usage.MeterEgressGB: 10}}, 10 |
| + with, err := priceBillingCalculation(st2, h2) |
| + if err != nil || with.Arithmetic != got.Arithmetic || with.Usage != got.Usage || with.Invoice.TotalCents != got.Invoice.TotalCents || with.CountedDays != nil || with.Allowance.Partial { |
| + t.Fatalf("a full month with daily meters prices differently: %+v %v", with, err) |
| + } |
| +} |
| + |
| func TestBillingCalculationOwnsSnapshotAndDeterministicBytes(t *testing.T) { |
| st, h := calculationFixture(plans.PlanPersonal, 1) |
| st.Meters[usage.MeterEgressGB] = 10 |
| @@ -274,6 +469,22 @@ func TestPrepareBillingCalculationRefusesUnprovenMeters(t *testing.T) { |
| if _, err := prepareBillingCalculation(context.Background(), nil, st, h, st.UpdatedAt); !errors.Is(err, usage.ErrEvidenceChanged) { |
| t.Fatalf("missing source store allowed: %v", err) |
| } |
| + // A partly covered month goes through the same gate. Being billed on |
| + // some of its days is not a reason to bill on unproven or incomplete |
| + // evidence for the month. |
| + h[0].EffectiveAt = time.Date(2026, 9, 16, 0, 0, 0, 0, time.UTC) |
| + st.Days, st.Meters[usage.MeterInstanceHours] = septemberDays(usage.MeterInstanceHours, 100) |
| + if priced, err := priceBillingCalculation(st, h); err != nil || !priced.Allowance.Partial { |
| + t.Fatalf("fixture: the partial month should price: %v", err) |
| + } |
| + if _, err := prepareBillingCalculation(context.Background(), release.NewDirStore(t.TempDir()), st, h, st.UpdatedAt); !errors.Is(err, usage.ErrEvidenceChanged) { |
| + t.Fatalf("a partial month with unproven evidence was prepared: %v", err) |
| + } |
| + st.SourceDays[usage.SourceDurability] = 29 |
| + st.Evidence = nil |
| + if _, err := prepareBillingCalculation(context.Background(), release.NewDirStore(t.TempDir()), st, h, st.UpdatedAt); err == nil { |
| + t.Fatal("a partial month with an incomplete meter was prepared") |
| + } |
| } |
| func TestBillingCalculationAssignmentTimezoneStable(t *testing.T) { |
diff --git a/internal/console/billinginvoice.go b/internal/console/billinginvoice.go| index d413e3285..bc94209de 100644 |
| --- a/internal/console/billinginvoice.go |
| +++ b/internal/console/billinginvoice.go |
| @@ -193,7 +193,7 @@ func billingInvoiceQuiet(err error) bool { |
| } |
| func billingInvoiceDiagnostic(err error) string { |
| - for _, known := range []error{errBillingStatementAbsent, errBillingStatementOpen, errBillingModeMismatch, errBillingInvoiceHeld, errBillingInvoiceMismatch, errBillingLeaseHeld, errBillingLeaseLost, errBillingReview, errBillingTransition, errBillingOperation, errBillingDays, plans.ErrAssignmentHistory, usage.ErrEvidenceChanged} { |
| + for _, known := range []error{errBillingStatementAbsent, errBillingStatementOpen, errBillingModeMismatch, errBillingInvoiceHeld, errBillingInvoiceMismatch, errBillingLeaseHeld, errBillingLeaseLost, errBillingReview, errBillingTransition, errBillingOperation, errBillingDays, errBillingDaysMismatch, plans.ErrAssignmentHistory, usage.ErrEvidenceChanged} { |
| if errors.Is(err, known) { |
| return known.Error() |
| } |
diff --git a/internal/console/durabilityfold.go b/internal/console/durabilityfold.go| index 64ace6a4f..528439465 100644 |
| --- a/internal/console/durabilityfold.go |
| +++ b/internal/console/durabilityfold.go |
| @@ -5,6 +5,7 @@ import ( |
| "errors" |
| "log" |
| "os" |
| + "sort" |
| "time" |
| "github.com/carlosframework/platform/internal/usage" |
| @@ -19,12 +20,20 @@ import ( |
| // and does what it published PROVE it recorded every request. |
| // |
| // The proof is the ledger's own contract (usage.DurabilityDay.Scope) over |
| -// the box's expected span of the day, and it is strict: exactly one ledger |
| -// generation, no conflict, no gap, no recording failure, no unknown |
| -// outcome, the day closed, and coverage attested for EVERY account. A day |
| -// attested for everyone except some accounts does not count either, |
| -// because a statement's SourceDays is one number shared by every account, |
| -// and it cannot say "complete for all but these". That errs toward holding. |
| +// the box's expected span of the day: exactly one ledger generation, no |
| +// conflict, no gap, no recording failure, no unknown outcome, the day |
| +// closed, and coverage attested. |
| +// |
| +// Attestation is scoped. A box may attest that every replication request |
| +// it made went through its ledger EXCEPT for the databases of some named |
| +// accounts, which an uninstrumented process on the box replicates (the |
| +// box-wide litestream unit, for routes that are not hibernating exec |
| +// routes). Such a day is complete for every account the box did not |
| +// exclude and incomplete for each one it did. The fold reports the |
| +// excluded accounts beside the day's verdict, and a statement's durability |
| +// days are counted per account (usageSweep.DurabilityExcluded): an excluded |
| +// account's replication-write meter does not complete, so its month holds, |
| +// and nobody else's month holds because of it. |
| // |
| // No production box attests today (the root litestream unit replicates |
| // unit-backed routes outside the ledger), so this source reads zero days |
| @@ -43,19 +52,31 @@ import ( |
| // than the box began serving, ran its replication unmetered for the |
| // difference and must not be left out of the question. It holds the day. |
| // |
| -// The spans are compared to the hour. A box registers each source on that |
| -// source's first tick, each with its own first-seen instant, and they can |
| -// differ by seconds on a healthy box; first-seen is write-once, so an |
| -// exact comparison would hold a new box's first day for good over a |
| -// registration race. In practice the host backdates a late ledger's |
| +// The spans are compared with a tolerance (durabilityRegistrationSlack). A |
| +// box registers each source on that source's first tick, each with its own |
| +// first-seen instant, and they can differ by seconds on a healthy box; |
| +// first-seen is write-once, so an exact comparison would hold a new box's |
| +// first day for good over a registration race. A tolerance, not a |
| +// comparison of clock hours: two seconds either side of the top of an hour |
| +// are still two seconds. In practice the host backdates a late ledger's |
| // first-seen to the box's earliest evidence (host.boxEarliestEvidence), so |
| // a ledger that opened late is mostly caught further down, by its missing |
| // day files. This check is the backstop for a registration that was not |
| // backdated. |
| -func (s *Server) foldDurabilityDay(ctx context.Context, cands []sourceCandidate, active map[string][]sourceCandidate, d time.Time, date string, lifecycleOK bool, sweep *usageSweep, touch func(string) map[string]float64) bool { |
| +// durabilityRegistrationSlack is how much later than the box's other |
| +// sources its durability source may have been first seen and still be the |
| +// same boot. Sources register seconds apart; this is minutes, and far |
| +// short of the hour or more a ledger that opened late would show. |
| +const durabilityRegistrationSlack = 5 * time.Minute |
| + |
| +// excluded is the union, over every box, of the accounts that box's |
| +// attestation left out for the day. It is meaningful only when complete is |
| +// true. |
| +func (s *Server) foldDurabilityDay(ctx context.Context, cands []sourceCandidate, active map[string][]sourceCandidate, d time.Time, date string, lifecycleOK bool, sweep *usageSweep, touch func(string) map[string]float64) (complete bool, excluded []string) { |
| // No candidate proves nothing: a deployment that has never published a |
| // ledger has no evidence that nothing replicated. |
| - complete := lifecycleOK && len(cands) > 0 |
| + complete = lifecycleOK && len(cands) > 0 |
| + out := map[string]bool{} |
| // Every box another per-box source expected that day must be expected |
| // by the durability source too, for at least the same span. A box with |
| // no ledger directory, or one whose ledger was first registered after |
| @@ -76,7 +97,7 @@ func (s *Server) foldDurabilityDay(ctx context.Context, cands []sourceCandidate, |
| continue |
| } |
| dFrom, dTo, dExpected := cand.window(usage.SourceDurability, d) |
| - if !dExpected || dFrom.Truncate(time.Hour).After(from.Truncate(time.Hour)) || dTo.Truncate(time.Hour).Before(to.Truncate(time.Hour)) { |
| + if !dExpected || dFrom.Sub(from) > durabilityRegistrationSlack || to.Sub(dTo) > durabilityRegistrationSlack { |
| complete = false |
| } |
| } |
| @@ -108,8 +129,12 @@ func (s *Server) foldDurabilityDay(ctx context.Context, cands []sourceCandidate, |
| // not exist. The rows still fold; the record is contradicted. |
| log.Printf("console: sweep usage: %s exists outside box %s's lifecycle interval", key, box) |
| complete = false |
| - } else if ok, unmetered, _ := day.Scope(wFrom, wTo); !ok || len(unmetered) > 0 { |
| + } else if ok, unmetered, _ := day.Scope(wFrom, wTo); !ok { |
| complete = false |
| + } else { |
| + for _, acct := range unmetered { |
| + out[acct] = true |
| + } |
| } |
| for _, snap := range day.Epochs { |
| content, derr := snap.Decode() |
| @@ -130,5 +155,9 @@ func (s *Server) foldDurabilityDay(ctx context.Context, cands []sourceCandidate, |
| } |
| } |
| } |
| - return complete |
| + for acct := range out { |
| + excluded = append(excluded, acct) |
| + } |
| + sort.Strings(excluded) |
| + return complete, excluded |
| } |
diff --git a/internal/console/economicspass.go b/internal/console/economicspass.go| index cc8c7470a..32a7c57d3 100644 |
| --- a/internal/console/economicspass.go |
| +++ b/internal/console/economicspass.go |
| @@ -473,6 +473,11 @@ type usageSweep struct { |
| // minute, which the scan-time owner cannot show. Counted only on days |
| // SourceOwnership counts, and gating MeterCPUCoreHours alone. |
| SourceDays map[string]int |
| + // DurabilityExcluded counts, per account, the days SourceDays counts |
| + // for usage.SourceDurability on which a box's scoped attestation left |
| + // that account out (durabilityfold.go). The account's own statement |
| + // carries SourceDays[SourceDurability] less this number. |
| + DurabilityExcluded map[string]int |
| // EgressMetered marks every box that has EVER written a usage/edge day |
| // (discovered via ListDirs on usage.EdgeBoxesPrefix, independent of |
| // [from, to]) — a box that reports metrics but has no edge component at |
| @@ -560,14 +565,15 @@ func (s *Server) sweepUsage(ctx context.Context, from, to time.Time, routes []re |
| } |
| sweep := &usageSweep{ |
| - PerAccount: map[string]map[string]float64{}, |
| - PerAccountDay: map[string]map[string]map[string]float64{}, |
| - PerBox: map[string]economics.BoxLoad{}, |
| - BoxDays: map[string]int{}, |
| - SourceDays: map[string]int{}, |
| - EgressMetered: map[string]bool{}, |
| - Census: map[string]dayCensus{}, |
| - Storage: map[string]dayStorage{}, |
| + PerAccount: map[string]map[string]float64{}, |
| + PerAccountDay: map[string]map[string]map[string]float64{}, |
| + PerBox: map[string]economics.BoxLoad{}, |
| + BoxDays: map[string]int{}, |
| + SourceDays: map[string]int{}, |
| + DurabilityExcluded: map[string]int{}, |
| + EgressMetered: map[string]bool{}, |
| + Census: map[string]dayCensus{}, |
| + Storage: map[string]dayStorage{}, |
| } |
| // A box is metered for egress the moment it has EVER written a day — |
| // independent of the swept period, matching the struct's "wrote any |
| @@ -821,7 +827,7 @@ func (s *Server) sweepUsage(ctx context.Context, from, to time.Time, routes []re |
| // Replication writes (durabilityfold.go): folded as a lower bound |
| // whatever the proof says; the day counts only when every expected |
| // box's ledger attests it box-wide. |
| - durabilityOK := s.foldDurabilityDay(ctx, durabilityCands, map[string][]sourceCandidate{usage.SourceMetrics: metricsCands, usage.SourceEdge: edgeCands}, d, date, lifecycleOK, sweep, touch) |
| + durabilityOK, durabilityExcluded := s.foldDurabilityDay(ctx, durabilityCands, map[string][]sourceCandidate{usage.SourceMetrics: metricsCands, usage.SourceEdge: edgeCands}, d, date, lifecycleOK, sweep, touch) |
| // Storage is read ONCE here and cached (sweep.Storage), the same way |
| // census is, so foldStorageAndCensus folds the EXACT bytes this |
| @@ -896,6 +902,9 @@ func (s *Server) sweepUsage(ctx context.Context, from, to time.Time, routes []re |
| } |
| if durabilityOK { |
| sweep.SourceDays[usage.SourceDurability]++ |
| + for _, acct := range durabilityExcluded { |
| + sweep.DurabilityExcluded[acct]++ |
| + } |
| } |
| if storageOK { |
| sweep.SourceDays[usage.SourceStorage]++ |
| @@ -1291,21 +1300,21 @@ func (s *Server) statementsPass(ctx context.Context, now time.Time) error { |
| return nil |
| } |
| // Read the whole window first. Metering began at the OLDEST month |
| - // that has a platform statement, not at the first month without |
| - // one: a console that was down for a month or two leaves a gap, and |
| - // stopping at the gap would hide every held month behind it for |
| - // good. |
| + // that has any statement, not at the first month without a |
| + // platform statement: a console that was down for a month or two |
| + // leaves a gap, and stopping at the gap would hide every held month |
| + // behind it for good. |
| states := make([]statementMonth, statementCatchUpMonths+1) |
| oldest := 0 |
| for back := 2; back <= statementCatchUpMonths; back++ { |
| start := firstOfThisMonth.AddDate(0, -back, 0) |
| states[back] = s.statementMonthState(ctx, usage.MonthOf(start), start.AddDate(0, 1, 0), accounts) |
| - if states[back] != statementMonthAbsent { |
| + if states[back] == statementMonthOpen || states[back] == statementMonthSettled { |
| oldest = back |
| } |
| } |
| for back := 2; back <= oldest; back++ { |
| - if states[back] == statementMonthSettled { |
| + if states[back] == statementMonthSettled || states[back] == statementMonthUnknown { |
| continue |
| } |
| // Open, or a gap: a month inside the metered range that no |
| @@ -1316,6 +1325,7 @@ func (s *Server) statementsPass(ctx context.Context, now time.Time) error { |
| if back > statementCatchUpWarnMonths { |
| log.Printf("console: statements pass: %s has been open for %d months and cannot be billed; someone must repair what holds it", month, back) |
| } |
| + s.statementAbsent.Delete(month) |
| if err := s.statementsForMonth(ctx, month, start, start.AddDate(0, 1, -1), nowUTC, true, routes); err != nil { |
| log.Printf("console: statements pass: %s (catch-up): %v", month, err) |
| } |
| @@ -1338,84 +1348,118 @@ const ( |
| type statementMonth int |
| const ( |
| - // No platform statement: no pass ever wrote this month. |
| + // No statement of any account: no pass ever wrote this month. |
| statementMonthAbsent statementMonth = iota |
| - // Some statement is open, missing for an account that existed, or |
| - // unreadable: the month must be swept again. |
| + // Some statement is open or missing for an account that existed: the |
| + // month must be swept again. |
| statementMonthOpen |
| - // Every statement the month owes is there and closed. |
| + // Every statement the month owes is there and closed (or is corrupt, |
| + // which a sweep cannot repair). |
| statementMonthSettled |
| + // A statement could not be read this pass. Nothing is concluded and |
| + // nothing is swept; the next pass reads again. |
| + statementMonthUnknown |
| ) |
| // statementMonthState classifies one month for the catch-up. monthEnd is |
| // the first instant after it. |
| // |
| -// The platform account's statement is written by every pass, so its absence |
| -// means no pass wrote the month. Otherwise every live account is checked, |
| -// not only the platform's: a closing pass that wrote the platform statement |
| -// and failed on one account's leaves that account's month open, or with no |
| -// statement at all, and either must be retried. An account created after |
| -// the month ended owes it no statement. A statement that cannot be read or |
| -// parsed is neither absent nor closed: it is logged and the month is swept |
| -// again (the sweep is idempotent and never rewrites a closed statement), so |
| -// one bad object cannot hide the months behind it. |
| +// Every live account's statement is read, the platform's included. A month |
| +// is Absent only when none exists: the platform's statement alone missing |
| +// (its write failed in the pass that wrote a tenant's) is an open month, |
| +// not an unmetered one. A statement that is open, or missing for an |
| +// account that existed before the month ended, makes the month Open. An |
| +// account created after the month ended owes it no statement. |
| +// |
| +// A statement that will not parse is not absent, not open and not closed. |
| +// A sweep cannot repair it (usage.UpsertStatement refuses to overwrite |
| +// what it cannot read), so sweeping the month for it every pass would |
| +// spend a full month's reads for nothing, for up to three years. It is |
| +// logged once per process, by key, for a person to repair, and counts as |
| +// present. A statement that could not be fetched at all (a store error) |
| +// concludes nothing: the month is Unknown this pass. |
| // |
| // An account deleted since drops out of Store.Accounts and is not looked |
| // for: its open month is not retried and cannot be billed, which errs in |
| // the customer's favour. |
| // |
| -// A statement seen closed is remembered for the life of the process and |
| -// not read again: closure is irreversible (usage.UpsertStatement), and |
| -// without this every pass would re-read every account's statement for |
| -// every metered month. Anything that ever reopens a statement must run in |
| -// a fresh process or clear s.statementClosed. |
| +// Two things are remembered for the life of the process, so that a steady |
| +// state costs no reads. A statement seen closed is not read again: closure |
| +// is irreversible (usage.UpsertStatement). A month seen with no statement |
| +// at all is not read again until this process sweeps it: no pass writes a |
| +// month two or more back except the catch-up, which only sweeps months |
| +// inside the metered range. Anything that ever reopens or deletes a closed |
| +// statement must run in a fresh process or clear s.statementClosed. |
| func (s *Server) statementMonthState(ctx context.Context, month string, monthEnd time.Time, accounts []Account) statementMonth { |
| - read := func(acct string) (found, closed bool, err error) { |
| + if _, ok := s.statementAbsent.Load(month); ok { |
| + return statementMonthAbsent |
| + } |
| + const ( |
| + missing = iota |
| + open |
| + closed |
| + corrupt |
| + unread |
| + ) |
| + read := func(acct string) int { |
| key := usage.StatementKey(acct, month) |
| if _, ok := s.statementClosed.Load(key); ok { |
| - return true, true, nil |
| + return closed |
| } |
| raw, _, err := s.cfg.Releases.Get(ctx, key) |
| if errors.Is(err, os.ErrNotExist) { |
| - return false, false, nil |
| + return missing |
| } |
| if err != nil { |
| - return false, false, err |
| + log.Printf("console: statements pass: catch-up: read %s: %v", key, err) |
| + return unread |
| } |
| var st usage.Statement |
| if err := json.Unmarshal(raw, &st); err != nil { |
| - return false, false, err |
| + if _, logged := s.statementCorrupt.LoadOrStore(key, struct{}{}); !logged { |
| + log.Printf("console: statements pass: catch-up: %s cannot be parsed and no sweep can repair it; someone must fix or remove it: %v", key, err) |
| + } |
| + return corrupt |
| } |
| if st.Closed { |
| s.statementClosed.Store(key, struct{}{}) |
| + return closed |
| } |
| - return true, st.Closed, nil |
| + return open |
| } |
| - found, closed, err := read(release.OpsAccount) |
| - switch { |
| - case err != nil: |
| - log.Printf("console: statements pass: catch-up: %s: platform statement unreadable: %v", month, err) |
| - return statementMonthOpen |
| - case !found: |
| - return statementMonthAbsent |
| - case !closed: |
| - return statementMonthOpen |
| + platform := read(release.OpsAccount) |
| + if platform == unread { |
| + return statementMonthUnknown |
| } |
| + // any: some statement exists, so the month was metered. owed: a |
| + // statement the month owes is open or missing. A missing platform |
| + // statement is owed whenever the month was metered at all. |
| + any := platform != missing |
| + owed := platform == open || platform == missing |
| for _, a := range accounts { |
| if a.Sqid == release.OpsAccount { |
| continue |
| } |
| - found, closed, err := read(a.Sqid) |
| - switch { |
| - case err != nil: |
| - log.Printf("console: statements pass: catch-up: %s: statement of %s unreadable: %v", month, a.Sqid, err) |
| - return statementMonthOpen |
| - case found && !closed: |
| - return statementMonthOpen |
| - case !found && a.CreatedAt.Before(monthEnd): |
| - return statementMonthOpen |
| + switch read(a.Sqid) { |
| + case unread: |
| + return statementMonthUnknown |
| + case open: |
| + any, owed = true, true |
| + case closed, corrupt: |
| + any = true |
| + case missing: |
| + if a.CreatedAt.Before(monthEnd) { |
| + owed = true |
| + } |
| } |
| } |
| + switch { |
| + case !any: |
| + s.statementAbsent.Store(month, struct{}{}) |
| + return statementMonthAbsent |
| + case owed: |
| + return statementMonthOpen |
| + } |
| return statementMonthSettled |
| } |
| @@ -1565,7 +1609,7 @@ func (s *Server) statementsForMonth(ctx context.Context, month string, start, en |
| Month: month, |
| Meters: mm, |
| CoveredDays: coveredDays, |
| - SourceDays: sourceDays, |
| + SourceDays: accountSourceDays(sourceDays, sweep.DurabilityExcluded[acct]), |
| Days: sweep.PerAccountDay[acct], |
| DaysInMonth: daysInMonth, |
| UpdatedAt: now, |
| @@ -1586,6 +1630,22 @@ func (s *Server) statementsForMonth(ctx context.Context, month string, start, en |
| return nil |
| } |
| +// accountSourceDays is the month's source days as one account's statement |
| +// carries them: the shared counts, with the durability source reduced by |
| +// the days a box's scoped attestation excluded this account. An account no |
| +// box excluded gets the shared map itself. |
| +func accountSourceDays(shared map[string]int, durabilityExcluded int) map[string]int { |
| + if durabilityExcluded <= 0 { |
| + return shared |
| + } |
| + out := make(map[string]int, len(shared)) |
| + for source, days := range shared { |
| + out[source] = days |
| + } |
| + out[usage.SourceDurability] = max(0, out[usage.SourceDurability]-durabilityExcluded) |
| + return out |
| +} |
| + |
| // billCEDayGrace is how many days into the FOLLOWING month a closed month's |
| // bill must have been pulled before billPass stops re-pulling it forever — |
| // CE's late-arriving line items settle by then, so a pull on or after this |
diff --git a/internal/console/meterfolds_test.go b/internal/console/meterfolds_test.go| index 99d2e0b1a..c0f2851c6 100644 |
| --- a/internal/console/meterfolds_test.go |
| +++ b/internal/console/meterfolds_test.go |
| @@ -12,6 +12,8 @@ import ( |
| "github.com/carlosframework/platform/internal/awsbilling" |
| "github.com/carlosframework/platform/internal/metrics" |
| "github.com/carlosframework/platform/internal/objstore" |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| "github.com/carlosframework/platform/internal/usage" |
| ) |
| @@ -279,11 +281,19 @@ func (c *foldClock) now() time.Time { return c.t } |
| // publishDurabilityDay runs a real ledger on box for 2026-09-17, records |
| // attempts through it and publishes the closed day into the fixture bucket. |
| func publishDurabilityDay(t *testing.T, fx *consoleFixture, box string, attested bool, record func(bind func(usage.ReplicationAttribution) *usage.DurabilityClient, at time.Time)) { |
| + t.Helper() |
| + publishScopedDurabilityDay(t, fx, box, attested, nil, record) |
| +} |
| + |
| +// publishScopedDurabilityDay is publishDurabilityDay with the attestation's |
| +// exclusion set: the accounts whose databases an uninstrumented process on |
| +// the box replicates. |
| +func publishScopedDurabilityDay(t *testing.T, fx *consoleFixture, box string, attested bool, unmetered []string, record func(bind func(usage.ReplicationAttribution) *usage.DurabilityClient, at time.Time)) { |
| t.Helper() |
| ctx := context.Background() |
| day := time.Date(2026, 9, 17, 0, 0, 0, 0, time.UTC) |
| clock := &foldClock{t: day.Add(-time.Hour)} |
| - l, err := usage.OpenDurabilityLedger(t.TempDir(), box, clock.now, usage.DurabilityLedgerOptions{AllClientsInstrumented: attested}) |
| + l, err := usage.OpenDurabilityLedger(t.TempDir(), box, clock.now, usage.DurabilityLedgerOptions{AllClientsInstrumented: attested, Unmetered: unmetered}) |
| if err != nil { |
| t.Fatal(err) |
| } |
| @@ -328,6 +338,68 @@ func tenantWrites(t *testing.T) func(func(usage.ReplicationAttribution) *usage.D |
| } |
| } |
| +// A scoped attestation. The box attests every replication request it made |
| +// went through its ledger, except for the databases of one account, which |
| +// the box-wide replicator handles outside it. The day is complete for the |
| +// account the box vouched for and incomplete for the one it excluded: the |
| +// excluded account's meter cannot complete and its month holds, and nobody |
| +// else holds because of it. |
| +func TestDurabilityFoldScopedAttestationHoldsOnlyTheExcludedAccount(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + if err := fx.st.CreateAccount(testAccountID(), foldAcctB, "Always on", "owner-b@example.test", time.Date(2026, 8, 20, 0, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + publishScopedDurabilityDay(t, fx, "box1", true, []string{foldAcctB}, tenantWrites(t)) |
| + if err := fx.srv.statementsPass(ctx, time.Date(2026, 9, 20, 6, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + vouched, excluded := septemberStatement(t, fx, foldAcctA), septemberStatement(t, fx, foldAcctB) |
| + if got := vouched.SourceDays[usage.SourceDurability]; got != 1 { |
| + t.Errorf("the vouched-for account: durability source days = %d, want 1", got) |
| + } |
| + if got := excluded.SourceDays[usage.SourceDurability]; got != 0 { |
| + t.Errorf("the excluded account: durability source days = %d, want 0", got) |
| + } |
| + // Every other source is the same number for both: only the one meter |
| + // the exclusion is about differs. |
| + for source, days := range vouched.SourceDays { |
| + if source != usage.SourceDurability && excluded.SourceDays[source] != days { |
| + t.Errorf("source %s: %d for the excluded account, %d for the other", source, excluded.SourceDays[source], days) |
| + } |
| + } |
| + // The platform account, which no box excluded, is vouched for too. |
| + if got := septemberStatement(t, fx, fixtureAcct).SourceDays[usage.SourceDurability]; got != 1 { |
| + t.Errorf("an account outside the exclusion set: durability source days = %d, want 1", got) |
| + } |
| + // Two boxes, and only one of them excludes the account: it is excluded |
| + // for the day. One box cannot vouch for another's replicator. |
| + fx2 := newConsoleFixture(t) |
| + if err := fx2.st.CreateAccount(testAccountID(), foldAcctB, "Always on", "owner-b@example.test", time.Date(2026, 8, 20, 0, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + publishScopedDurabilityDay(t, fx2, "box1", true, nil, tenantWrites(t)) |
| + publishScopedDurabilityDay(t, fx2, "box2", true, []string{foldAcctB}, func(func(usage.ReplicationAttribution) *usage.DurabilityClient, time.Time) {}) |
| + if err := fx2.srv.statementsPass(ctx, time.Date(2026, 9, 20, 6, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if got := septemberStatement(t, fx2, foldAcctB).SourceDays[usage.SourceDurability]; got != 0 { |
| + t.Errorf("excluded by one of two boxes: durability source days = %d, want 0", got) |
| + } |
| + if got := septemberStatement(t, fx2, foldAcctA).SourceDays[usage.SourceDurability]; got != 1 { |
| + t.Errorf("vouched for by both boxes: durability source days = %d, want 1", got) |
| + } |
| + // An unattested box still proves nothing for anyone. |
| + fx3 := newConsoleFixture(t) |
| + publishScopedDurabilityDay(t, fx3, "box1", false, []string{foldAcctB}, tenantWrites(t)) |
| + if err := fx3.srv.statementsPass(ctx, time.Date(2026, 9, 20, 6, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if got := septemberStatement(t, fx3, foldAcctA).SourceDays[usage.SourceDurability]; got != 0 { |
| + t.Errorf("an unattested box: durability source days = %d, want 0", got) |
| + } |
| +} |
| + |
| // An attested, closed box-day folds its billable writes and counts as a |
| // source day; the statement carries the meter. |
| func TestDurabilityFoldCountsAnAttestedDay(t *testing.T) { |
| @@ -516,6 +588,17 @@ func TestDurabilityFoldHoldsBeforeABoxRegisteredItsLedger(t *testing.T) { |
| if got := run(day.Add(9*time.Hour + 5*time.Second)); got != same-1 { |
| t.Errorf("durability source days = %d, want %d: the ledger registered an hour after the box began serving", got, same-1) |
| } |
| + // Nor is ten minutes. |
| + if got := run(edgeFrom.Add(10 * time.Minute)); got != same-1 { |
| + t.Errorf("durability source days = %d, want %d: the ledger registered ten minutes after the box began serving", got, same-1) |
| + } |
| + // The same race of seconds, across the top of an hour: the edge at |
| + // 08:59:59, the ledger at 09:00:01. Still one boot. |
| + edgeFrom = day.Add(8*time.Hour + 59*time.Minute + 59*time.Second) |
| + same = run(edgeFrom) |
| + if got := run(day.Add(9*time.Hour + time.Second)); got != same { |
| + t.Errorf("durability source days = %d, want %d: two seconds across an hour boundary held a new box's first day", got, same) |
| + } |
| } |
| // reopenStatement puts a closed statement back to open, as a failed write |
| @@ -692,8 +775,8 @@ func TestStatementCatchUpWalksPastAnUnreadableMonth(t *testing.T) { |
| t.Fatal(err) |
| } |
| accounts, _ := fx.st.Accounts() |
| - if got := fx.srv.statementMonthState(ctx, "2026-10", time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC), accounts); got != statementMonthOpen { |
| - t.Fatalf("an unreadable platform statement = %v, want open (never absent)", got) |
| + if got := fx.srv.statementMonthState(ctx, "2026-10", time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC), accounts); got == statementMonthAbsent || got == statementMonthUnknown { |
| + t.Fatalf("an unparseable platform statement = %v: the month was metered and must read as open or settled", got) |
| } |
| if err := fx.srv.statementsPass(ctx, time.Date(2026, 12, 5, 6, 0, 0, 0, time.UTC)); err != nil { |
| t.Fatal(err) |
| @@ -703,6 +786,105 @@ func TestStatementCatchUpWalksPastAnUnreadableMonth(t *testing.T) { |
| } |
| } |
| +// The first metered month. Its pass wrote the tenant's statement, open, and |
| +// failed to write the platform's. The console was then down for a month. A |
| +// walk that takes "no platform statement" for "never metered" puts |
| +// September outside the metered range for good, with the tenant's statement |
| +// and all its evidence sitting there. |
| +func TestStatementCatchUpFindsAMonthWhosePlatformStatementIsMissing(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + fx.srv.cfg.StoreEgressReportPrefix = foldReportPrefix |
| + if err := fx.st.CreateAccount(testAccountID(), foldAcctA, "Tenant", "owner@example.test", time.Date(2026, 8, 20, 0, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + openOwner(t, fx, foldBucketA, foldAcctA, "shop") |
| + // September's only pass, mid-month: both statements written open. |
| + if err := fx.srv.statementsPass(ctx, time.Date(2026, 9, 20, 6, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if septemberStatement(t, fx, foldAcctA).Closed { |
| + t.Fatal("fixture: September should be open") |
| + } |
| + // The platform statement's write is the one that failed. |
| + if err := fx.rel.Delete(ctx, usage.StatementKey(fixtureAcct, "2026-09")); err != nil { |
| + t.Fatal(err) |
| + } |
| + accounts, _ := fx.st.Accounts() |
| + if got := fx.srv.statementMonthState(ctx, "2026-09", time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC), accounts); got != statementMonthOpen { |
| + t.Fatalf("a month with a tenant statement and no platform statement = %v, want open", got) |
| + } |
| + putCURDelivery(t, fx, "exec-1", curLine("l1", "INV-1", "EUW1-DataTransfer-Out-Bytes", foldBucketA, "2026-09-03T10:00:00Z", "2", "0.09", "0.18")) |
| + // Back in November: October was never written at all. |
| + if err := fx.srv.statementsPass(ctx, time.Date(2026, 11, 5, 6, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if !statementExists(t, fx, fixtureAcct, "2026-09") || !septemberStatement(t, fx, fixtureAcct).Closed || !septemberStatement(t, fx, foldAcctA).Closed { |
| + t.Fatal("September, the oldest month, with only a tenant statement, was never swept") |
| + } |
| +} |
| + |
| +// What the walk remembers and what it refuses to conclude. |
| +func TestStatementMonthStateEdges(t *testing.T) { |
| + fx := closedSeptember(t) |
| + ctx := context.Background() |
| + accounts, _ := fx.st.Accounts() |
| + state := func(month string) statementMonth { |
| + start, _ := time.Parse("2006-01", month) |
| + return fx.srv.statementMonthState(ctx, month, start.AddDate(0, 1, 0), accounts) |
| + } |
| + // A month nobody wrote is absent, and remembered so it is not read |
| + // again every pass. |
| + if got := state("2026-05"); got != statementMonthAbsent { |
| + t.Fatalf("an unwritten month = %v", got) |
| + } |
| + if _, ok := fx.srv.statementAbsent.Load("2026-05"); !ok { |
| + t.Fatal("an absent month is not remembered") |
| + } |
| + // A tenant statement that will not parse cannot be repaired by a |
| + // sweep. It does not make the month open, and it is logged once. |
| + key := usage.StatementKey(foldAcctA, "2026-09") |
| + _, etag, err := fx.rel.Get(ctx, key) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + fresh := &Server{cfg: fx.srv.cfg, store: fx.srv.store} |
| + if _, err := fx.rel.Put(ctx, key, []byte("{not json"), etag); err != nil { |
| + t.Fatal(err) |
| + } |
| + start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) |
| + for i := 0; i < 2; i++ { |
| + if got := fresh.statementMonthState(ctx, "2026-09", start.AddDate(0, 1, 0), accounts); got != statementMonthSettled { |
| + t.Fatalf("a month whose only fault is an unparseable tenant statement = %v, want settled (a sweep cannot repair it)", got) |
| + } |
| + } |
| + if _, ok := fresh.statementCorrupt.Load(key); !ok { |
| + t.Fatal("the unparseable statement was not recorded as reported") |
| + } |
| + // A store error concludes nothing: not absent, not open, not settled. |
| + broken := &Server{cfg: fx.srv.cfg, store: fx.srv.store} |
| + broken.cfg.Releases = failingGets{Store: fx.rel, fail: usage.StatementKey(fixtureAcct, "2026-09")} |
| + if got := broken.statementMonthState(ctx, "2026-09", start.AddDate(0, 1, 0), accounts); got != statementMonthUnknown { |
| + t.Fatalf("a month whose platform statement could not be fetched = %v, want unknown", got) |
| + } |
| + if _, ok := broken.statementAbsent.Load("2026-09"); ok { |
| + t.Fatal("a store error was remembered as an absent month") |
| + } |
| +} |
| + |
| +// failingGets fails every Get of one key with a store error. |
| +type failingGets struct { |
| + release.Store |
| + fail string |
| +} |
| + |
| +func (f failingGets) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + if key == f.fail { |
| + return nil, "", errors.New("store had a bad moment") |
| + } |
| + return f.Store.Get(ctx, key) |
| +} |
| + |
| // Every billed meter's days sum to its monthly total, whatever source it |
| // came from. A partly covered month is billed from the days, so they must |
| // account for the month. |
| @@ -710,18 +892,40 @@ func TestStatementDaysSumToTheMonthlyMeters(t *testing.T) { |
| fx := newConsoleFixture(t) |
| ctx := context.Background() |
| putStoreJSON(t, fx.rel, metrics.DayKey("box1", "2026-09-02"), watchedMetricsDay("box1", "2026-09-02")) |
| - putStoreJSON(t, fx.rel, metrics.DayKey("box1", "2026-09-09"), watchedMetricsDay("box1", "2026-09-09")) |
| + // One awake unit on the ninth, for instance hours. |
| + reg, err := registry.Open(fx.regPath) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := reg.Upsert(registry.Route{Host: "a.example.com", Kind: "instance", Status: "ready", App: fixtureApp, Version: "v1"}); err != nil { |
| + t.Fatal(err) |
| + } |
| + ninth := watchedMetricsDay("box1", "2026-09-09") |
| + for _, hour := range []string{"09", "10"} { |
| + agg := ninth.Hours[hour] |
| + agg.Units = map[string]map[string]metrics.Agg{"carlos-app@a.example.com.service": {"cpu": {Avg: 0.5, N: 60}, "mem_bytes": {Avg: 1e9, N: 60}}} |
| + ninth.Hours[hour] = agg |
| + } |
| + putStoreJSON(t, fx.rel, metrics.DayKey("box1", "2026-09-09"), ninth) |
| putStoreJSON(t, fx.rel, usage.EdgeDayKey("box1", "2026-09-02"), watchedEdgeDay("box1", "2026-09-02")) |
| putStoreJSON(t, fx.rel, usage.EdgeDayKey("box1", "2026-09-11"), watchedEdgeDay("box1", "2026-09-11")) |
| for _, date := range []string{"2026-09-05", "2026-09-06"} { |
| putStoreJSON(t, fx.rel, usage.StorageDayKey(date), usage.StorageDay{V: 1, Date: date, |
| Apps: map[string]usage.AppStorage{fixtureAcct + "/" + fixtureApp: {BucketBytes: 3_000_000_000}}}) |
| } |
| - if err := fx.srv.statementsPass(ctx, time.Date(2026, 9, 20, 6, 0, 0, 0, time.UTC)); err != nil { |
| + // The two sources that arrive by other roads: a box's replication |
| + // ledger, and the AWS report, across two days and two buckets. |
| + fx.srv.cfg.StoreEgressReportPrefix = foldReportPrefix |
| + publishDurabilityDay(t, fx, "box1", true, tenantWrites(t)) |
| + openOwner(t, fx, foldBucketA, foldAcctA, "shop") |
| + putCURDelivery(t, fx, "exec-1", |
| + curLine("l1", "INV-1", "EUW1-DataTransfer-Out-Bytes", foldBucketA, "2026-09-03T10:00:00Z", "2", "0.09", "0.18")+ |
| + curLine("l2", "INV-1", "EU-APS2-AWS-Out-Bytes", foldBucketA, "2026-09-21T23:00:00Z", "5", "0.02", "0.10")) |
| + if err := fx.srv.statementsPass(ctx, time.Date(2026, 10, 5, 6, 0, 0, 0, time.UTC)); err != nil { |
| t.Fatal(err) |
| } |
| - seen := 0 |
| - for _, acct := range []string{fixtureAcct} { |
| + seen := map[string]bool{} |
| + for _, acct := range []string{fixtureAcct, foldAcctA} { |
| st := septemberStatement(t, fx, acct) |
| for _, meter := range monthlyBillingMeters { |
| sum := 0.0 |
| @@ -732,11 +936,20 @@ func TestStatementDaysSumToTheMonthlyMeters(t *testing.T) { |
| t.Errorf("%s %s: days sum to %v, the month says %v", acct, meter, sum, st.Meters[meter]) |
| } |
| if st.Meters[meter] != 0 { |
| - seen++ |
| + seen[meter] = true |
| + } |
| + } |
| + for date := range st.Days { |
| + if !strings.HasPrefix(date, "2026-09-") { |
| + t.Errorf("%s: a day outside the month: %s", acct, date) |
| } |
| } |
| } |
| - if seen == 0 { |
| - t.Fatal("fixture recorded no billed meter at all; the invariant was not exercised") |
| + // Every billed meter was exercised, so the invariant is pinned for each |
| + // source that feeds one. |
| + for _, meter := range monthlyBillingMeters { |
| + if !seen[meter] { |
| + t.Errorf("the fixture recorded no %s: its days are not pinned by this test", meter) |
| + } |
| } |
| } |
diff --git a/internal/console/server.go b/internal/console/server.go| index c41a0dbf4..b1646f944 100644 |
| --- a/internal/console/server.go |
| +++ b/internal/console/server.go |
| @@ -697,6 +697,10 @@ type Server struct { |
| // statements catch-up does not re-read them every pass. See |
| // statementMonthState. |
| statementClosed sync.Map |
| + // statementAbsent remembers months seen with no statement at all, and |
| + // statementCorrupt the unparseable statement keys already logged. |
| + statementAbsent sync.Map |
| + statementCorrupt sync.Map |
| // claims folds the deployment's domain claims for the read paths — the |
| // Settings render and the domains list door — reading only the claims |
| // whose bucket stamp moved since the last fold (#173). Process-lifetime |
diff --git a/internal/usage/statement.go b/internal/usage/statement.go| index 490357a7d..2ee76d5c4 100644 |
| --- a/internal/usage/statement.go |
| +++ b/internal/usage/statement.go |
| @@ -27,7 +27,10 @@ type Statement struct { |
| Meters map[string]float64 `json:"meters"` |
| CoveredDays int `json:"covered_days"` |
| // SourceDays counts, per collection source, how many UTC days of the |
| - // month produced that source's object. CoveredDays answers "did any |
| + // month produced that source's object. It is this account's count: the |
| + // durability source is reduced by the days a box's scoped attestation |
| + // excluded this account, so one account's exclusion holds only its own |
| + // replication-write meter. CoveredDays answers "did any |
| // source report", which is the modelling question; this answers "did |
| // THIS meter's source report every day", which is the billing one. |
| // Absent (a statement written before this field existed) reads as |
| @@ -35,8 +38,10 @@ type Statement struct { |
| SourceDays map[string]int `json:"source_days,omitempty"` |
| // Days is the month's billed meters by UTC day ("2006-01-02" -> meter |
| // -> value), so a month a paid plan covered only partly can be billed |
| - // on the usage that fell inside the paid days and nothing else. Each |
| - // meter's days sum to its entry in Meters. Absent on a statement |
| + // on the usage that fell inside the paid days and nothing else. Only |
| + // the five billed meters are recorded by day (CPU, memory, requests, |
| + // backhaul and fleet meters are not), and each of those meters' days |
| + // sum to its entry in Meters. Absent on a statement |
| // written before the field existed; such a statement can bill a whole |
| // month and never a partial one. |
| Days map[string]map[string]float64 `json:"days,omitempty"` |