amadan
Bill a partly covered month on its paid days; fix review round three
Partial-month billing. A first month that starts mid-month, a month a subscription ends in and a lapse were refused quietly and never billed. plans.AllowanceForMonth now returns the paid spans of the month and the allowance weighted over them. It still refuses a history it cannot read, a month with no paid span, and paid spans that disagree on terms or the house marker. Statements carry each meter by UTC day, summing to the monthly totals. A partly covered month bills only the days whose 24 hours lie inside one paid span, records which days it counted in the frozen calculation, and is held (billing_statement_days_unusable) when the statement has no usable days. Completeness is still whole-month. Bucket ownership (Astra R3, Opus 5). An absent bucket proves nothing about credentials, so a ledger claims completeness only when the provisioner can also list every principal whose policy names the bucket and finds none. The AWS provisioner cannot yet, so on AWS no ledger claims completeness and store transfer holds. Completeness dates from the observation, not from the grant block. Teardown records a legacy store, incomplete, before it destroys anything, so a failed revocation leaves an open interval the next owner of the name inherits. Statement catch-up (Astra R4, Opus 7). The walk no longer stops at the first month with no platform statement: metering began at the oldest month in the window that has one, and a gap inside that range is built. A statement that cannot be read is logged and the month swept again, never treated as absent. An account that existed in the month and has no statement is open. Statements seen closed are remembered for the life of the process. Durability fold (Opus 4). Spans are compared to the hour, so a registration race of seconds no longer holds a new box's first day for good. Tests the reviewers showed were weak now fail without their fixes: the classifier table asserts that every case named as a hold holds the month, with fee and discount lines added; the evidence test covers a boundary that moves before the month; the catch-up tests cross real gaps and nine months. A host test that raced its temp dir removal against the ledger's late writer (2 failures in 30) waits for it. Verified: go build, go vet, gofmt and make test pass outside the sandbox (60 packages ok). Review round four has not run.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
22 files changed,
+1080
−154
internal/awsbilling/classify_test.go+18 −1internal/console/billingcalculation.go+90 −2internal/console/billingcalculation_test.go+108 −2internal/console/billinginvoice.go+2 −2internal/console/billinginvoice_test.go+42 −4internal/console/billingrenewal_test.go+22 −7internal/console/durabilityfold.go+12 −1internal/console/economicspass.go+130 −38internal/console/economicsview.go+1 −1internal/console/meterfolds_test.go+236 −45internal/console/server.go+4 −0internal/console/storeegressfold.go+9 −2internal/host/durability_test.go+6 −0internal/objstore/memprovisioner.go+22 −0internal/objstore/ownership.go+18 −5internal/objstore/ownership_test.go+145 −13internal/objstore/provisioner.go+16 −0internal/objstore/walk.go+37 −11internal/plans/month.go+68 −5internal/plans/month_test.go+75 −11internal/usage/billable_test.go+9 −1internal/usage/statement.go+10 −3
diff --git a/internal/awsbilling/classify_test.go b/internal/awsbilling/classify_test.go| index 0b4781abd..75f510af0 100644 |
| --- a/internal/awsbilling/classify_test.go |
| +++ b/internal/awsbilling/classify_test.go |
| @@ -2,6 +2,7 @@ package awsbilling |
| import ( |
| "math/big" |
| + "strings" |
| "testing" |
| "time" |
| ) |
| @@ -78,6 +79,16 @@ func TestChargedS3TransferTable(t *testing.T) { |
| l.LineItemType = "Refund" |
| return l |
| }, want{false, 0, ReasonTransferAdjustment}}, |
| + {"a positive fee on transfer holds", func() Line { |
| + l := s3Line(t, "EUW1-DataTransfer-Out-Bytes", "0", "0.25") |
| + l.LineItemType = "Fee" |
| + return l |
| + }, want{false, 0, ReasonTransferAdjustment}}, |
| + {"a discount on transfer holds", func() Line { |
| + l := s3Line(t, "EUW1-DataTransfer-Out-Bytes", "3", "-0.027") |
| + l.LineItemType = "EdpDiscount" |
| + return l |
| + }, want{false, 0, ReasonTransferAdjustment}}, |
| {"a credit on storage is not transfer", func() Line { |
| l := s3Line(t, "EUW1-TimedStorage-ByteHrs", "0", "-0.01") |
| l.LineItemType = "Credit" |
| @@ -147,11 +158,17 @@ func TestChargedS3TransferTable(t *testing.T) { |
| if charged && IsHold(reason) { |
| t.Errorf("%s: a counted line cannot be a hold", c.name) |
| } |
| + // A case that says it holds must hold the month, not merely go |
| + // uncounted: an uncounted line that does not hold is a line the |
| + // tenant's bill silently ignores. |
| + if strings.Contains(c.name, "holds") != IsHold(reason) { |
| + t.Errorf("%s: reason %q, IsHold=%v", c.name, reason, IsHold(reason)) |
| + } |
| } |
| } |
| func TestIsHold(t *testing.T) { |
| - for _, r := range []string{ReasonUnknownTransfer, ReasonNoResource, ReasonNotHourly, ReasonBadAmount} { |
| + for _, r := range []string{ReasonUnknownTransfer, ReasonUnexpectedCharge, ReasonTransferAdjustment, ReasonNoResource, ReasonNotHourly, ReasonBadAmount} { |
| if !IsHold(r) { |
| t.Errorf("%s must hold", r) |
| } |
diff --git a/internal/console/billingcalculation.go b/internal/console/billingcalculation.go| index 2056dd9a5..e5f8e59ae 100644 |
| --- a/internal/console/billingcalculation.go |
| +++ b/internal/console/billingcalculation.go |
| @@ -38,6 +38,11 @@ type billingCalculation struct { |
| Invoice plans.InvoiceResult `json:"invoice"` |
| Lines []plans.OverageLine `json:"lines"` |
| TaxCode string `json:"tax_code"` |
| + // CountedDays is set only for a month a paid plan covered partly |
| + // (Allowance.Partial): the UTC days whose usage was billed, each one |
| + // wholly inside a paid interval. Usage on every other day of the month |
| + // was not billed. |
| + CountedDays []string `json:"counted_days,omitempty"` |
| } |
| func prepareBillingCalculation(ctx context.Context, store release.Store, statement usage.Statement, history []plans.Assignment, now time.Time) (*billingCalculation, error) { |
| @@ -76,6 +81,18 @@ func priceBillingCalculation(statement usage.Statement, history []plans.Assignme |
| if len(allowance.Assignments) == 0 { |
| return nil, plans.ErrAssignmentHistory |
| } |
| + // A month covered end to end bills its monthly totals, as it always |
| + // has. A month covered partly bills only the days that lie wholly |
| + // inside a paid interval, against the allowance for those intervals. |
| + billed, arithmetic := statement.Meters, "base_units_floor_v1" |
| + var counted []string |
| + if allowance.Partial { |
| + billed, counted, err = billingUsageOnPaidDays(statement, allowance.Covered) |
| + if err != nil { |
| + return nil, err |
| + } |
| + arithmetic = "base_units_floor_whole_paid_days_v1" |
| + } |
| var amounts [5]int64 |
| for i, meter := range monthlyBillingMeters { |
| scale := int64(1_000_000_000) |
| @@ -85,7 +102,7 @@ func priceBillingCalculation(statement usage.Statement, history []plans.Assignme |
| if meter == usage.MeterDurabilityWrites1k { |
| scale = 1000 |
| } |
| - amounts[i], err = billingBaseUnits(statement.Meters[meter], scale) |
| + amounts[i], err = billingBaseUnits(billed[meter], scale) |
| if err != nil { |
| return nil, err |
| } |
| @@ -103,7 +120,78 @@ func priceBillingCalculation(statement usage.Statement, history []plans.Assignme |
| return nil, err |
| } |
| digest := sha256.Sum256(raw) |
| - return &billingCalculation{V: 1, Arithmetic: "base_units_floor_v1", Currency: "eur", Statement: statement, StatementSHA256: hex.EncodeToString(digest[:]), Allowance: *allowance, Usage: used, Overage: *overage, Invoice: *invoice, Lines: lines}, nil |
| + return &billingCalculation{V: 1, Arithmetic: arithmetic, Currency: "eur", Statement: statement, StatementSHA256: hex.EncodeToString(digest[:]), Allowance: *allowance, Usage: used, Overage: *overage, Invoice: *invoice, Lines: lines, CountedDays: counted}, nil |
| +} |
| + |
| +// errBillingDays means a partly covered month cannot be billed from this |
| +// statement: it carries no daily meters, or they do not add up to its |
| +// monthly totals. The month holds; it is never billed on the monthly total, |
| +// which would charge usage from outside the paid days. |
| +var errBillingDays = errors.New("billing_statement_days_unusable") |
| + |
| +// billingUsageOnPaidDays returns each billed meter summed over the UTC days |
| +// of the month that lie WHOLLY inside one paid interval, and those days. |
| +// The day a plan starts or ends on is not counted: its usage cannot be |
| +// split, and the rounding goes to the customer. |
| +// |
| +// The statement's days must account for its monthly totals, meter by |
| +// meter. A statement written before daily meters existed, or one whose days |
| +// disagree with its totals, is refused. |
| +func billingUsageOnPaidDays(statement usage.Statement, covered []plans.CoveredInterval) (map[string]float64, []string, error) { |
| + start, err := time.Parse("2006-01", statement.Month) |
| + if err != nil { |
| + return nil, nil, errBillingDays |
| + } |
| + end := start.AddDate(0, 1, 0) |
| + if statement.Days == nil { |
| + // No day recorded anything. That is a usable statement only when |
| + // the month recorded nothing at all. |
| + for _, meter := range monthlyBillingMeters { |
| + if statement.Meters[meter] != 0 { |
| + return nil, nil, errBillingDays |
| + } |
| + } |
| + } |
| + sums := map[string]float64{} |
| + for date, mm := range statement.Days { |
| + day, perr := time.Parse("2006-01-02", date) |
| + if perr != nil || day.Before(start) || !day.Before(end) { |
| + return nil, nil, errBillingDays |
| + } |
| + for meter, v := range mm { |
| + if v < 0 || math.IsNaN(v) || math.IsInf(v, 0) { |
| + return nil, nil, errBillingMeter |
| + } |
| + sums[meter] += v |
| + } |
| + } |
| + for _, meter := range monthlyBillingMeters { |
| + total := statement.Meters[meter] |
| + if diff := math.Abs(sums[meter] - total); diff > 1e-9*math.Max(1, math.Abs(total)) { |
| + return nil, nil, errBillingDays |
| + } |
| + } |
| + out := map[string]float64{} |
| + var counted []string |
| + for day := start; day.Before(end); day = day.AddDate(0, 0, 1) { |
| + next := day.AddDate(0, 0, 1) |
| + inside := false |
| + for _, iv := range covered { |
| + if !iv.From.After(day) && !iv.To.Before(next) { |
| + inside = true |
| + break |
| + } |
| + } |
| + if !inside { |
| + continue |
| + } |
| + date := usage.DayOf(day) |
| + counted = append(counted, date) |
| + for _, meter := range monthlyBillingMeters { |
| + out[meter] += statement.Days[date][meter] |
| + } |
| + } |
| + return out, counted, nil |
| } |
| func billingBaseUnits(value float64, scale int64) (int64, error) { |
diff --git a/internal/console/billingcalculation_test.go b/internal/console/billingcalculation_test.go| index 3849e9c4f..590ac1edd 100644 |
| --- a/internal/console/billingcalculation_test.go |
| +++ b/internal/console/billingcalculation_test.go |
| @@ -121,10 +121,116 @@ func TestBillingCalculationRejectsInvalidUsageAndHistory(t *testing.T) { |
| if _, err := priceBillingCalculation(st, h); err == nil { |
| t.Fatal("Free account got an overage invoice") |
| } |
| + // A month the plan covered only partly is priced on its whole paid |
| + // days, never on the monthly total: with usage recorded for the month |
| + // and no daily breakdown, it holds. |
| st, h = calculationFixture(plans.PlanPersonal, 1) |
| h[0].EffectiveAt = h[0].EffectiveAt.Add(time.Hour) |
| - if _, err := priceBillingCalculation(st, h); err == nil { |
| - t.Fatal("uncovered assignment interval got priced") |
| + st.Meters[usage.MeterInstanceHours] = 500 |
| + if _, err := priceBillingCalculation(st, h); !errors.Is(err, errBillingDays) { |
| + t.Fatalf("a partly covered month with no daily meters: %v", err) |
| + } |
| +} |
| + |
| +// septemberDays spreads perDay of one meter over every day of September. |
| +func septemberDays(meter string, perDay float64) (map[string]map[string]float64, float64) { |
| + days := map[string]map[string]float64{} |
| + total := 0.0 |
| + for d := 1; d <= 30; d++ { |
| + days[time.Date(2026, 9, d, 0, 0, 0, 0, time.UTC).Format("2006-01-02")] = map[string]float64{meter: perDay} |
| + total += perDay |
| + } |
| + return days, total |
| +} |
| + |
| +// A first month that starts on the sixteenth: usage on the fifteen whole |
| +// paid days is billed against half the month's allowance, and the usage of |
| +// the first half of the month, when the account was not on the plan, is |
| +// not billed at all. |
| +func TestBillingCalculationBillsOnlyThePaidDaysOfAFirstMonth(t *testing.T) { |
| + st, h := calculationFixture(plans.PlanBusiness, 10) |
| + h[0].EffectiveAt = time.Date(2026, 9, 16, 0, 0, 0, 0, time.UTC) |
| + st.Days, st.Meters[usage.MeterInstanceHours] = septemberDays(usage.MeterInstanceHours, 100) |
| + got, err := priceBillingCalculation(st, h) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if !got.Allowance.Partial || got.Arithmetic != "base_units_floor_whole_paid_days_v1" { |
| + t.Fatalf("not priced as a partial month: %+v", got.Allowance) |
| + } |
| + if len(got.CountedDays) != 15 || got.CountedDays[0] != "2026-09-16" || got.CountedDays[14] != "2026-09-30" { |
| + t.Fatalf("counted days %v", got.CountedDays) |
| + } |
| + // 15 days x 100 h used; 10 members x 170 h x 15/30 allowed; 650 h over |
| + // at EUR 0.010 an hour. |
| + if got.Usage.AwakeSeconds != 1500*3600 || got.Allowance.Resources.AwakeSeconds != 850*3600 || got.Invoice.TotalCents != 650 { |
| + t.Fatalf("usage %d allowance %d total %d", got.Usage.AwakeSeconds, got.Allowance.Resources.AwakeSeconds, got.Invoice.TotalCents) |
| + } |
| + // The whole month on the same plan bills the monthly total, as before. |
| + st2, h2 := calculationFixture(plans.PlanBusiness, 10) |
| + st2.Days, st2.Meters[usage.MeterInstanceHours] = septemberDays(usage.MeterInstanceHours, 100) |
| + whole, err := priceBillingCalculation(st2, h2) |
| + if err != nil || whole.Allowance.Partial || whole.CountedDays != nil || whole.Arithmetic != "base_units_floor_v1" || whole.Usage.AwakeSeconds != 3000*3600 { |
| + t.Fatalf("whole month: %+v %v", whole, err) |
| + } |
| +} |
| + |
| +// The day a plan starts or ends on is not counted: its usage cannot be |
| +// split, so it goes to the customer. |
| +func TestBillingCalculationNeverCountsABoundaryDay(t *testing.T) { |
| + // Starts at noon on the sixteenth. |
| + st, h := calculationFixture(plans.PlanBusiness, 10) |
| + h[0].EffectiveAt = time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) |
| + st.Days, st.Meters[usage.MeterInstanceHours] = septemberDays(usage.MeterInstanceHours, 100) |
| + got, err := priceBillingCalculation(st, h) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(got.CountedDays) != 14 || got.CountedDays[0] != "2026-09-17" { |
| + t.Fatalf("counted days %v", got.CountedDays) |
| + } |
| + if got.Usage.AwakeSeconds != 1400*3600 || got.Allowance.Resources.AwakeSeconds != 2_958_000 { |
| + t.Fatalf("usage %d allowance %d", got.Usage.AwakeSeconds, got.Allowance.Resources.AwakeSeconds) |
| + } |
| + // Cancelled at noon on the sixteenth: the first fifteen days count. |
| + st, h = calculationFixture(plans.PlanBusiness, 10) |
| + ended := h[0] |
| + ended.Revision, ended.Status, ended.EffectiveAt = 2, "canceled", time.Date(2026, 9, 16, 12, 0, 0, 0, time.UTC) |
| + h = append(h, ended) |
| + st.Days, st.Meters[usage.MeterInstanceHours] = septemberDays(usage.MeterInstanceHours, 100) |
| + got, err = priceBillingCalculation(st, h) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(got.CountedDays) != 15 || got.CountedDays[0] != "2026-09-01" || got.CountedDays[14] != "2026-09-15" { |
| + t.Fatalf("counted days after a cancellation %v", got.CountedDays) |
| + } |
| +} |
| + |
| +// Daily meters that do not account for the monthly total, or name a day |
| +// outside the month, hold a partly covered month. |
| +func TestBillingCalculationHoldsAPartialMonthOnUnusableDays(t *testing.T) { |
| + mid := time.Date(2026, 9, 16, 0, 0, 0, 0, time.UTC) |
| + st, h := calculationFixture(plans.PlanBusiness, 10) |
| + h[0].EffectiveAt = mid |
| + st.Days, st.Meters[usage.MeterInstanceHours] = septemberDays(usage.MeterInstanceHours, 100) |
| + st.Meters[usage.MeterInstanceHours] += 50 // the days no longer add up |
| + if _, err := priceBillingCalculation(st, h); !errors.Is(err, errBillingDays) { |
| + t.Fatalf("days that do not sum to the total: %v", err) |
| + } |
| + st, h = calculationFixture(plans.PlanBusiness, 10) |
| + h[0].EffectiveAt = mid |
| + st.Days = map[string]map[string]float64{"2026-10-01": {usage.MeterInstanceHours: 1}} |
| + st.Meters[usage.MeterInstanceHours] = 1 |
| + if _, err := priceBillingCalculation(st, h); !errors.Is(err, errBillingDays) { |
| + t.Fatalf("a day outside the month: %v", err) |
| + } |
| + // A partly covered month that recorded nothing bills nothing. |
| + st, h = calculationFixture(plans.PlanBusiness, 10) |
| + h[0].EffectiveAt = mid |
| + got, err := priceBillingCalculation(st, h) |
| + if err != nil || got.Invoice.TotalCents != 0 { |
| + t.Fatalf("an idle partial month: %+v %v", got, err) |
| } |
| } |
diff --git a/internal/console/billinginvoice.go b/internal/console/billinginvoice.go| index b35f755fc..d413e3285 100644 |
| --- a/internal/console/billinginvoice.go |
| +++ b/internal/console/billinginvoice.go |
| @@ -184,7 +184,7 @@ func billingInvoiceMonths(first int64, now time.Time) []string { |
| } |
| func billingInvoiceQuiet(err error) bool { |
| - for _, known := range []error{errBillingStatementAbsent, errBillingStatementOpen, errBillingPending, errBillingReview, errBillingLeaseHeld, errBillingModeMismatch, plans.ErrAssignmentHistory, usage.ErrEvidenceChanged} { |
| + for _, known := range []error{errBillingStatementAbsent, errBillingStatementOpen, errBillingPending, errBillingReview, errBillingLeaseHeld, errBillingModeMismatch, errBillingDays, plans.ErrAssignmentHistory, usage.ErrEvidenceChanged} { |
| if errors.Is(err, known) { |
| return true |
| } |
| @@ -193,7 +193,7 @@ func billingInvoiceQuiet(err error) bool { |
| } |
| func billingInvoiceDiagnostic(err error) string { |
| - for _, known := range []error{errBillingStatementAbsent, errBillingStatementOpen, errBillingModeMismatch, errBillingInvoiceHeld, errBillingInvoiceMismatch, errBillingLeaseHeld, errBillingLeaseLost, errBillingReview, errBillingTransition, errBillingOperation, plans.ErrAssignmentHistory, usage.ErrEvidenceChanged} { |
| + for _, known := range []error{errBillingStatementAbsent, errBillingStatementOpen, errBillingModeMismatch, errBillingInvoiceHeld, errBillingInvoiceMismatch, errBillingLeaseHeld, errBillingLeaseLost, errBillingReview, errBillingTransition, errBillingOperation, errBillingDays, plans.ErrAssignmentHistory, usage.ErrEvidenceChanged} { |
| if errors.Is(err, known) { |
| return known.Error() |
| } |
diff --git a/internal/console/billinginvoice_test.go b/internal/console/billinginvoice_test.go| index 21153f4c6..390d7fe72 100644 |
| --- a/internal/console/billinginvoice_test.go |
| +++ b/internal/console/billinginvoice_test.go |
| @@ -693,13 +693,15 @@ func TestInvoiceEngineHoldsBeforeAnyProviderCall(t *testing.T) { |
| } |
| want = errBillingStatementAbsent |
| case "partial_first_month": |
| - // Paid from the 15th: the month is neither billed from the |
| - // whole statement (that would charge pre-paid Free usage) nor |
| - // waived; it is refused until segmented usage exists. |
| + // Paid from the 15th, and the statement carries no daily |
| + // meters: the month is neither billed from the whole |
| + // statement (that would charge pre-paid Free usage) nor |
| + // waived. It holds. With daily meters it is billed on its |
| + // paid days (TestInvoiceEngineBillsAPartialFirstMonthOnItsPaidDays). |
| if _, err := ix.st.db.Exec(`UPDATE billing_assignments SET effective_at=? WHERE account_id=?`, time.Date(2026, 9, 15, 0, 0, 0, 0, time.UTC).Unix(), ix.acct.ID); err != nil { |
| t.Fatal(err) |
| } |
| - want = plans.ErrAssignmentHistory |
| + want = errBillingDays |
| case "mode_flip": |
| ix.srv.billing.cfg.LiveMode = true |
| want = errBillingModeMismatch |
| @@ -1520,3 +1522,39 @@ func TestInvoiceEngineChecksOwnerImmediatelyBeforePost(t *testing.T) { |
| t.Fatalf("POST without ownership: %+v", arrivals) |
| } |
| } |
| + |
| +// A first month paid from the 15th, with daily meters on the statement, is |
| +// invoiced through the whole engine: the sixteen whole paid days are billed |
| +// against sixteen thirtieths of the allowance, and the fourteen days before |
| +// the plan began are not billed. |
| +func TestInvoiceEngineBillsAPartialFirstMonthOnItsPaidDays(t *testing.T) { |
| + ix := newInvoiceFixture(t) |
| + if _, err := ix.st.db.Exec(`UPDATE billing_assignments SET effective_at=? WHERE account_id=?`, time.Date(2026, 9, 15, 0, 0, 0, 0, time.UTC).Unix(), ix.acct.ID); err != nil { |
| + t.Fatal(err) |
| + } |
| + ix.state.Days, ix.state.Meters[usage.MeterInstanceHours] = septemberDays(usage.MeterInstanceHours, 100) |
| + ix.putStatement(t) |
| + if err := ix.run(t); err != nil { |
| + t.Fatal(err) |
| + } |
| + rec, err := ix.st.billingMonth(ix.acct.ID, invoiceFixtureMonth) |
| + if err != nil || rec.State != "complete" { |
| + t.Fatalf("month record %+v %v", rec, err) |
| + } |
| + var calc billingCalculation |
| + if err := json.Unmarshal(rec.Calculation, &calc); err != nil { |
| + t.Fatal(err) |
| + } |
| + if !calc.Allowance.Partial || len(calc.CountedDays) != 16 || calc.CountedDays[0] != "2026-09-15" { |
| + t.Fatalf("counted days %v partial %v", calc.CountedDays, calc.Allowance.Partial) |
| + } |
| + // 16 days x 100 h used; Personal allows 85 h x 16/30; the rest at |
| + // EUR 0.010 an hour: 1554.67 cents, rounded half up by the existing |
| + // invoice total rule. |
| + if calc.Usage.AwakeSeconds != 1600*3600 || calc.Allowance.Resources.AwakeSeconds != 163_200 || calc.Invoice.TotalCents != 1555 { |
| + t.Fatalf("usage %d allowance %d total %d", calc.Usage.AwakeSeconds, calc.Allowance.Resources.AwakeSeconds, calc.Invoice.TotalCents) |
| + } |
| + if arrivals, _, _ := ix.fake.counts(); len(arrivals) == 0 { |
| + t.Fatal("a billable partial month sent nothing to the provider") |
| + } |
| +} |
diff --git a/internal/console/billingrenewal_test.go b/internal/console/billingrenewal_test.go| index 689d935d8..6fc60f127 100644 |
| --- a/internal/console/billingrenewal_test.go |
| +++ b/internal/console/billingrenewal_test.go |
| @@ -217,10 +217,13 @@ func TestBillingRenewalWebhookFiveSecondsLateIsDatedAtPeriodStart(t *testing.T) |
| t.Fatalf("allowance %+v, want %+v", got.Resources, quote.Allowances) |
| } |
| // The same history dated at observation, as before, leaves a five-second |
| - // gap the allowance must refuse. That gap is what the dating closes. |
| + // gap no assignment paid for. The month is then partly covered, and the |
| + // day of the gap is not a whole paid day. That gap is what the dating |
| + // closes. |
| history[1].EffectiveAt = observed |
| - if _, err := plans.AllowanceForMonth("2026-09", history); !errors.Is(err, plans.ErrAssignmentHistory) { |
| - t.Fatalf("observation-dated history accepted: %v", err) |
| + gapped, err := plans.AllowanceForMonth("2026-09", history) |
| + if err != nil || !gapped.Partial || len(gapped.Covered) != 2 || !gapped.Covered[0].To.Equal(renewalAt) || !gapped.Covered[1].From.Equal(observed) { |
| + t.Fatalf("observation-dated history read as fully covered: %+v %v", gapped, err) |
| } |
| // Redelivery and a later poll change nothing. |
| if err := fx.srv.reconcileBillingSubscription(context.Background(), "sub_live", "evt_renewed", observed.Add(time.Minute)); err != nil { |
| @@ -304,9 +307,16 @@ func TestBillingRenewalWithOtherChangesIsNotBackdated(t *testing.T) { |
| if got := renewalReviews(t, fx.st, acct.ID); !reflect.DeepEqual(got, []string{reason}) { |
| t.Fatalf("reviews %v, want %q", got, reason) |
| } |
| + // The month is never read as covered end to end, and it cannot |
| + // be frozen while the review filed above is open. |
| month := now.Format("2006-01") |
| - if _, err := plans.AllowanceForMonth(month, renewalHistory(t, fx.st, acct.ID)); !errors.Is(err, plans.ErrAssignmentHistory) { |
| - t.Fatalf("held month %s produced an allowance: %v", month, err) |
| + if got, err := plans.AllowanceForMonth(month, renewalHistory(t, fx.st, acct.ID)); err == nil && !got.Partial { |
| + t.Fatalf("held month %s read as fully covered: %+v", month, got) |
| + } else if err != nil && !errors.Is(err, plans.ErrAssignmentHistory) { |
| + t.Fatal(err) |
| + } |
| + if _, err := fx.st.freezeBillingMonth(acct.ID, month, []byte(`{"held":true}`)); !errors.Is(err, errBillingReview) { |
| + t.Fatalf("month %s froze under an open review: %v", month, err) |
| } |
| }) |
| } |
| @@ -405,8 +415,13 @@ func TestBillingDelayedRenewalNeverRewritesOrderOrFrozenMonths(t *testing.T) { |
| if got := renewalReviews(t, fx.st, acct.ID); !reflect.DeepEqual(got, []string{billingPeriodReviewReason(renewalAt.Unix())}) { |
| t.Fatalf("reviews: %v", got) |
| } |
| - if _, err := plans.AllowanceForMonth("2026-09", renewalHistory(t, fx.st, acct.ID)); !errors.Is(err, plans.ErrAssignmentHistory) { |
| - t.Fatalf("held month produced an allowance: %v", err) |
| + if got, err := plans.AllowanceForMonth("2026-09", renewalHistory(t, fx.st, acct.ID)); err == nil && !got.Partial { |
| + t.Fatalf("held month read as fully covered: %+v", got) |
| + } else if err != nil && !errors.Is(err, plans.ErrAssignmentHistory) { |
| + t.Fatal(err) |
| + } |
| + if _, err := fx.st.freezeBillingMonth(acct.ID, "2026-09", []byte(`{"held":true}`)); !errors.Is(err, errBillingReview) { |
| + t.Fatalf("month froze under an open review: %v", err) |
| } |
| }) |
| t.Run("frozen_month", func(t *testing.T) { |
diff --git a/internal/console/durabilityfold.go b/internal/console/durabilityfold.go| index 86c40d695..64ace6a4f 100644 |
| --- a/internal/console/durabilityfold.go |
| +++ b/internal/console/durabilityfold.go |
| @@ -42,6 +42,16 @@ import ( |
| // span. A box with no ledger directory, or whose ledger registered later |
| // than the box began serving, ran its replication unmetered for the |
| // difference and must not be left out of the question. It holds the day. |
| +// |
| +// The spans are compared to the hour. A box registers each source on that |
| +// source's first tick, each with its own first-seen instant, and they can |
| +// differ by seconds on a healthy box; first-seen is write-once, so an |
| +// exact comparison would hold a new box's first day for good over a |
| +// registration race. In practice the host backdates a late ledger's |
| +// first-seen to the box's earliest evidence (host.boxEarliestEvidence), so |
| +// a ledger that opened late is mostly caught further down, by its missing |
| +// day files. This check is the backstop for a registration that was not |
| +// backdated. |
| func (s *Server) foldDurabilityDay(ctx context.Context, cands []sourceCandidate, active map[string][]sourceCandidate, d time.Time, date string, lifecycleOK bool, sweep *usageSweep, touch func(string) map[string]float64) bool { |
| // No candidate proves nothing: a deployment that has never published a |
| // ledger has no evidence that nothing replicated. |
| @@ -66,7 +76,7 @@ func (s *Server) foldDurabilityDay(ctx context.Context, cands []sourceCandidate, |
| continue |
| } |
| dFrom, dTo, dExpected := cand.window(usage.SourceDurability, d) |
| - if !dExpected || dFrom.After(from) || dTo.Before(to) { |
| + if !dExpected || dFrom.Truncate(time.Hour).After(from.Truncate(time.Hour)) || dTo.Truncate(time.Hour).Before(to.Truncate(time.Hour)) { |
| complete = false |
| } |
| } |
| @@ -114,6 +124,7 @@ func (s *Server) foldDurabilityDay(ctx context.Context, cands []sourceCandidate, |
| for _, row := range rows { |
| if n := usage.BillableReplicationWrites(row); n > 0 { |
| touch(row.Account)[usage.MeterDurabilityWrites1k] += float64(n) / 1000 |
| + sweep.addDay(row.Account, date, usage.MeterDurabilityWrites1k, float64(n)/1000) |
| } |
| } |
| } |
diff --git a/internal/console/economicspass.go b/internal/console/economicspass.go| index 1f6a8f68c..cc8c7470a 100644 |
| --- a/internal/console/economicspass.go |
| +++ b/internal/console/economicspass.go |
| @@ -417,8 +417,13 @@ func (s *Server) readDayStorage(ctx context.Context, date string) dayStorage { |
| // every caller folds those in afterward via foldStorageAndCensus. |
| type usageSweep struct { |
| PerAccount map[string]map[string]float64 // account -> meter -> value |
| - PerBox map[string]economics.BoxLoad // box -> period load (Hours, per-account CPU/mem) |
| - BoxDays map[string]int // box -> days with any metrics rollup |
| + // PerAccountDay is the billed meters by UTC day (account -> date -> |
| + // meter -> value), the same additions as PerAccount, so a month a paid |
| + // plan covered only partly can be billed on its paid days alone |
| + // (usage.Statement.Days). Only the meters billing reads are kept. |
| + PerAccountDay map[string]map[string]map[string]float64 |
| + PerBox map[string]economics.BoxLoad // box -> period load (Hours, per-account CPU/mem) |
| + BoxDays map[string]int // box -> days with any metrics rollup |
| // Census carries each swept day's census, read once here (readDayCensus) |
| // and handed to foldStorageAndCensus so the same object is not fetched |
| // twice per day. Both callers fold over exactly the range they swept, so |
| @@ -556,6 +561,7 @@ func (s *Server) sweepUsage(ctx context.Context, from, to time.Time, routes []re |
| sweep := &usageSweep{ |
| PerAccount: map[string]map[string]float64{}, |
| + PerAccountDay: map[string]map[string]map[string]float64{}, |
| PerBox: map[string]economics.BoxLoad{}, |
| BoxDays: map[string]int{}, |
| SourceDays: map[string]int{}, |
| @@ -708,6 +714,7 @@ func (s *Server) sweepUsage(ctx context.Context, from, to time.Time, routes []re |
| instN = mem.N |
| } |
| mm[usage.MeterInstanceHours] += float64(instN) / 60 |
| + sweep.addDay(acct, date, usage.MeterInstanceHours, float64(instN)/60) |
| al := bl.ByAccount[acct] |
| al.CPUCoreHours += cpuCoreHours |
| al.MemGBHours += memGBHours |
| @@ -796,6 +803,7 @@ func (s *Server) sweepUsage(ctx context.Context, from, to time.Time, routes []re |
| } |
| mm := touch(acct) |
| mm[usage.MeterEgressGB] += float64(row.BytesOut) / 1e9 |
| + sweep.addDay(acct, date, usage.MeterEgressGB, float64(row.BytesOut)/1e9) |
| mm[usage.MeterRequests1k] += float64(row.Requests) / 1000 |
| // The hop SUBSET (usage.EdgeRow): traffic served through a |
| // region edge and backhauled from the home box, so the |
| @@ -905,6 +913,25 @@ func (s *Server) sweepUsage(ctx context.Context, from, to time.Time, routes []re |
| return sweep, nil |
| } |
| +// addDay adds v to one account's meter on one UTC day. Every billed meter |
| +// that lands in PerAccount also lands here, at the same call site. |
| +func (u *usageSweep) addDay(acct, date, meter string, v float64) { |
| + if u == nil || u.PerAccountDay == nil { |
| + return |
| + } |
| + days := u.PerAccountDay[acct] |
| + if days == nil { |
| + days = map[string]map[string]float64{} |
| + u.PerAccountDay[acct] = days |
| + } |
| + mm := days[date] |
| + if mm == nil { |
| + mm = map[string]float64{} |
| + days[date] = mm |
| + } |
| + mm[meter] += v |
| +} |
| + |
| // foldStorageAndCensus reads each UTC day's storage and census objects in |
| // [from, to] (both truncated to midnight, inclusive) and folds their meters |
| // into dst (account -> meter -> value) — MeterStorageGBMonth and |
| @@ -938,7 +965,7 @@ func (s *Server) sweepUsage(ctx context.Context, from, to time.Time, routes []re |
| // same day if the second Get failed (or succeeded) differently from the |
| // first. Nil, or a day either map does not hold, falls back to reading that |
| // day here — the function stays correct standing alone. |
| -func (s *Server) foldStorageAndCensus(ctx context.Context, from, to time.Time, dst map[string]map[string]float64, census map[string]dayCensus, storage map[string]dayStorage) error { |
| +func (s *Server) foldStorageAndCensus(ctx context.Context, from, to time.Time, dst map[string]map[string]float64, census map[string]dayCensus, storage map[string]dayStorage, days *usageSweep) error { |
| touch := func(acct string) map[string]float64 { |
| mm := dst[acct] |
| if mm == nil { |
| @@ -984,6 +1011,7 @@ func (s *Server) foldStorageAndCensus(ctx context.Context, from, to time.Time, d |
| } |
| bytesTotal := ap.BucketBytes + ap.StoreBytes |
| touch(acct)[usage.MeterStorageGBMonth] += (float64(bytesTotal) / 1e9) / float64(daysInMonth) |
| + days.addDay(acct, date, usage.MeterStorageGBMonth, (float64(bytesTotal)/1e9)/float64(daysInMonth)) |
| } |
| } // missing/unreadable = skip; sweepUsage already counted the day's coverage |
| @@ -1220,7 +1248,8 @@ const statementCloseDayGrace = usage.StatementCloseGraceDays |
| // leaves the month before it permanently Closed: false — still correct, |
| // still recomputed, but silently un-billable, since billing reads only |
| // closed statements. With a report configured, the catch-up at the end of |
| -// this function revisits every earlier month still open. |
| +// this function revisits every earlier month in its window that is still |
| +// open, was never written (a gap), or is missing an account's statement. |
| func (s *Server) statementsPass(ctx context.Context, now time.Time) error { |
| reg, err := s.openPassRegistry() |
| if err != nil { |
| @@ -1261,18 +1290,31 @@ func (s *Server) statementsPass(ctx context.Context, now time.Time) error { |
| log.Printf("console: statements pass: catch-up: %v", err) |
| return nil |
| } |
| + // Read the whole window first. Metering began at the OLDEST month |
| + // that has a platform statement, not at the first month without |
| + // one: a console that was down for a month or two leaves a gap, and |
| + // stopping at the gap would hide every held month behind it for |
| + // good. |
| + states := make([]statementMonth, statementCatchUpMonths+1) |
| + oldest := 0 |
| for back := 2; back <= statementCatchUpMonths; back++ { |
| start := firstOfThisMonth.AddDate(0, -back, 0) |
| - month := usage.MonthOf(start) |
| - exists, open := s.statementMonthState(ctx, month, accounts) |
| - if !exists { |
| - break // before this deployment metered anything |
| + states[back] = s.statementMonthState(ctx, usage.MonthOf(start), start.AddDate(0, 1, 0), accounts) |
| + if states[back] != statementMonthAbsent { |
| + oldest = back |
| } |
| - if !open { |
| + } |
| + for back := 2; back <= oldest; back++ { |
| + if states[back] == statementMonthSettled { |
| continue |
| } |
| + // Open, or a gap: a month inside the metered range that no |
| + // pass ever wrote. Its day objects are still there, so it is |
| + // built now. |
| + start := firstOfThisMonth.AddDate(0, -back, 0) |
| + month := usage.MonthOf(start) |
| if back > statementCatchUpWarnMonths { |
| - log.Printf("console: statements pass: %s has been held open for %d months and cannot be billed; someone must repair what holds it", month, back) |
| + log.Printf("console: statements pass: %s has been open for %d months and cannot be billed; someone must repair what holds it", month, back) |
| } |
| if err := s.statementsForMonth(ctx, month, start, start.AddDate(0, 1, -1), nowUTC, true, routes); err != nil { |
| log.Printf("console: statements pass: %s (catch-up): %v", month, err) |
| @@ -1283,49 +1325,98 @@ func (s *Server) statementsPass(ctx context.Context, now time.Time) error { |
| } |
| // statementCatchUpMonths bounds how far back statementsPass walks looking |
| -// for a month still open. The walk also stops at the first month with no |
| -// platform statement, which is where metering began. Three years is far |
| -// beyond any invoice delay; a month held longer than |
| -// statementCatchUpWarnMonths is logged on every pass. |
| +// for a month still open. Three years is far beyond any invoice delay. A |
| +// month open longer than statementCatchUpWarnMonths is logged on every |
| +// pass. A month still open when it leaves the window is never swept again: |
| +// that is the one way a month can now be abandoned, and it takes three |
| +// years of those log lines to get there. |
| const ( |
| statementCatchUpMonths = 36 |
| statementCatchUpWarnMonths = 6 |
| ) |
| -// statementMonthState reports whether the month has statements at all |
| -// (the platform account's is written by every pass, so it stands for |
| -// "this month was metered"), and whether any account's statement for it is |
| -// still open. Every account is checked, not only the platform's: a closing |
| -// pass that wrote the platform statement and failed on one account's |
| -// leaves that account's month open, and it must be retried. |
| -func (s *Server) statementMonthState(ctx context.Context, month string, accounts []Account) (exists, open bool) { |
| - read := func(acct string) (found, closed bool) { |
| - raw, _, err := s.cfg.Releases.Get(ctx, usage.StatementKey(acct, month)) |
| +type statementMonth int |
| + |
| +const ( |
| + // No platform statement: no pass ever wrote this month. |
| + statementMonthAbsent statementMonth = iota |
| + // Some statement is open, missing for an account that existed, or |
| + // unreadable: the month must be swept again. |
| + statementMonthOpen |
| + // Every statement the month owes is there and closed. |
| + statementMonthSettled |
| +) |
| + |
| +// statementMonthState classifies one month for the catch-up. monthEnd is |
| +// the first instant after it. |
| +// |
| +// The platform account's statement is written by every pass, so its absence |
| +// means no pass wrote the month. Otherwise every live account is checked, |
| +// not only the platform's: a closing pass that wrote the platform statement |
| +// and failed on one account's leaves that account's month open, or with no |
| +// statement at all, and either must be retried. An account created after |
| +// the month ended owes it no statement. A statement that cannot be read or |
| +// parsed is neither absent nor closed: it is logged and the month is swept |
| +// again (the sweep is idempotent and never rewrites a closed statement), so |
| +// one bad object cannot hide the months behind it. |
| +// |
| +// An account deleted since drops out of Store.Accounts and is not looked |
| +// for: its open month is not retried and cannot be billed, which errs in |
| +// the customer's favour. |
| +// |
| +// A statement seen closed is remembered for the life of the process and |
| +// not read again: closure is irreversible (usage.UpsertStatement), and |
| +// without this every pass would re-read every account's statement for |
| +// every metered month. Anything that ever reopens a statement must run in |
| +// a fresh process or clear s.statementClosed. |
| +func (s *Server) statementMonthState(ctx context.Context, month string, monthEnd time.Time, accounts []Account) statementMonth { |
| + read := func(acct string) (found, closed bool, err error) { |
| + key := usage.StatementKey(acct, month) |
| + if _, ok := s.statementClosed.Load(key); ok { |
| + return true, true, nil |
| + } |
| + raw, _, err := s.cfg.Releases.Get(ctx, key) |
| + if errors.Is(err, os.ErrNotExist) { |
| + return false, false, nil |
| + } |
| if err != nil { |
| - return false, false |
| + return false, false, err |
| } |
| var st usage.Statement |
| - if json.Unmarshal(raw, &st) != nil { |
| - return false, false |
| + if err := json.Unmarshal(raw, &st); err != nil { |
| + return false, false, err |
| } |
| - return true, st.Closed |
| - } |
| - found, closed := read(release.OpsAccount) |
| - if !found { |
| - return false, false |
| + if st.Closed { |
| + s.statementClosed.Store(key, struct{}{}) |
| + } |
| + return true, st.Closed, nil |
| } |
| - if !closed { |
| - return true, true |
| + found, closed, err := read(release.OpsAccount) |
| + switch { |
| + case err != nil: |
| + log.Printf("console: statements pass: catch-up: %s: platform statement unreadable: %v", month, err) |
| + return statementMonthOpen |
| + case !found: |
| + return statementMonthAbsent |
| + case !closed: |
| + return statementMonthOpen |
| } |
| for _, a := range accounts { |
| if a.Sqid == release.OpsAccount { |
| continue |
| } |
| - if f, c := read(a.Sqid); f && !c { |
| - return true, true |
| + found, closed, err := read(a.Sqid) |
| + switch { |
| + case err != nil: |
| + log.Printf("console: statements pass: catch-up: %s: statement of %s unreadable: %v", month, a.Sqid, err) |
| + return statementMonthOpen |
| + case found && !closed: |
| + return statementMonthOpen |
| + case !found && a.CreatedAt.Before(monthEnd): |
| + return statementMonthOpen |
| } |
| } |
| - return true, false |
| + return statementMonthSettled |
| } |
| // statementsForMonth folds one month's metrics, edge and storage days (plus |
| @@ -1388,7 +1479,7 @@ func (s *Server) statementsForMonth(ctx context.Context, month string, start, en |
| return fmt.Errorf("console: statements pass: %w", err) |
| } |
| meters := sweep.PerAccount // account -> meter -> value; folded into below |
| - if err := reader.foldStorageAndCensus(ctx, start, end, meters, sweep.Census, sweep.Storage); err != nil { |
| + if err := reader.foldStorageAndCensus(ctx, start, end, meters, sweep.Census, sweep.Storage, sweep); err != nil { |
| return fmt.Errorf("console: statements pass: %w", err) |
| } |
| fleetMM, fleetReadFailed, err := reader.fleetMeters(ctx, start, end) |
| @@ -1407,7 +1498,7 @@ func (s *Server) statementsForMonth(ctx context.Context, month string, start, en |
| } |
| // Object-store transfer, from the month's AWS Cost and Usage Report |
| // (storeegressfold.go). Whole-month: complete or not at all. |
| - storeEgress := reader.foldStoreEgress(ctx, start, meters) |
| + storeEgress := reader.foldStoreEgress(ctx, start, meters, sweep) |
| if storeEgress.Complete { |
| sweep.SourceDays[usage.SourceStoreEgress] = daysInMonth |
| } |
| @@ -1475,6 +1566,7 @@ func (s *Server) statementsForMonth(ctx context.Context, month string, start, en |
| Meters: mm, |
| CoveredDays: coveredDays, |
| SourceDays: sourceDays, |
| + Days: sweep.PerAccountDay[acct], |
| DaysInMonth: daysInMonth, |
| UpdatedAt: now, |
| Closed: closing, |
diff --git a/internal/console/economicsview.go b/internal/console/economicsview.go| index a42885af0..0c8dac652 100644 |
| --- a/internal/console/economicsview.go |
| +++ b/internal/console/economicsview.go |
| @@ -415,7 +415,7 @@ func (s *Server) buildEconomicsView(ctx context.Context, period string) (*Econom |
| if err != nil { |
| return nil, fmt.Errorf("console: load economics: %w", err) |
| } |
| - if err := s.foldStorageAndCensus(sweepCtx, from, to, sweep.PerAccount, sweep.Census, sweep.Storage); err != nil { |
| + if err := s.foldStorageAndCensus(sweepCtx, from, to, sweep.PerAccount, sweep.Census, sweep.Storage, nil); err != nil { |
| return nil, fmt.Errorf("console: load economics: %w", err) |
| } |
diff --git a/internal/console/meterfolds_test.go b/internal/console/meterfolds_test.go| index d44c122c3..99d2e0b1a 100644 |
| --- a/internal/console/meterfolds_test.go |
| +++ b/internal/console/meterfolds_test.go |
| @@ -4,11 +4,13 @@ import ( |
| "context" |
| "encoding/json" |
| "errors" |
| + "os" |
| "strings" |
| "testing" |
| "time" |
| "github.com/carlosframework/platform/internal/awsbilling" |
| + "github.com/carlosframework/platform/internal/metrics" |
| "github.com/carlosframework/platform/internal/objstore" |
| "github.com/carlosframework/platform/internal/usage" |
| ) |
| @@ -80,7 +82,7 @@ func near(a, b float64) bool { d := a - b; return d < 1e-9 && d > -1e-9 } |
| func openOwner(t *testing.T, fx *consoleFixture, bucket, acct, app string) { |
| t.Helper() |
| // minted: the ledger is created by the grant that mints the bucket. |
| - if err := objstore.OpenBucketOwner(context.Background(), fx.rel, bucket, acct, app, time.Date(2026, 8, 10, 0, 0, 0, 0, time.UTC), true); err != nil { |
| + if err := objstore.OpenBucketOwner(context.Background(), fx.rel, bucket, acct, app, time.Date(2026, 8, 10, 0, 0, 0, 0, time.UTC), time.Date(2026, 8, 10, 0, 0, 0, 0, time.UTC), true); err != nil { |
| t.Fatal(err) |
| } |
| } |
| @@ -113,6 +115,13 @@ func TestStoreEgressFoldBillsOnlyChargedOwnedTransfer(t *testing.T) { |
| if got := b.Meters[usage.MeterStoreEgressGB]; !near(got, 0.5) { |
| t.Errorf("account B store egress = %v GB, want 0.5", got) |
| } |
| + // The same bytes by day, for a month billed on its paid days alone. |
| + if got := a.Days["2026-09-03"][usage.MeterStoreEgressGB]; !near(got, 3.5) { |
| + t.Errorf("account A store egress on the 3rd = %v GB, want 3.5", got) |
| + } |
| + if got := b.Days["2026-09-04"][usage.MeterStoreEgressGB]; !near(got, 0.5) { |
| + t.Errorf("account B store egress on the 4th = %v GB, want 0.5", got) |
| + } |
| if got := a.SourceDays[usage.SourceStoreEgress]; got != 30 { |
| t.Errorf("store egress source days = %d, want 30", got) |
| } |
| @@ -170,7 +179,7 @@ func TestStoreEgressFoldHoldsOnUnownedBucket(t *testing.T) { |
| if a.Closed { |
| t.Fatal("the month closed over an ownership hole; it could never be billed") |
| } |
| - res := fx.srv.foldStoreEgress(ctx, foldSept, map[string]map[string]float64{}) |
| + res := fx.srv.foldStoreEgress(ctx, foldSept, map[string]map[string]float64{}, nil) |
| if res.Reason != storeEgressUnowned || len(res.Unowned) != 1 || res.Unowned[0] != foldBucketB { |
| t.Fatalf("hold reason %q unowned %v", res.Reason, res.Unowned) |
| } |
| @@ -203,7 +212,7 @@ func TestStoreEgressFoldWaitsForTheInvoice(t *testing.T) { |
| if !near(a.Meters[usage.MeterStoreEgressGB], 2) || a.MeterComplete(usage.MeterStoreEgressGB) || a.Closed { |
| t.Fatalf("uninvoiced month: meters=%v days=%d closed=%v", a.Meters, a.SourceDays[usage.SourceStoreEgress], a.Closed) |
| } |
| - if res := fx.srv.foldStoreEgress(ctx, foldSept, map[string]map[string]float64{}); res.Reason != storeEgressNotFinal || !res.HoldClose { |
| + if res := fx.srv.foldStoreEgress(ctx, foldSept, map[string]map[string]float64{}, nil); res.Reason != storeEgressNotFinal || !res.HoldClose { |
| t.Fatalf("reason %q hold %v", res.Reason, res.HoldClose) |
| } |
| } |
| @@ -216,7 +225,7 @@ func TestStoreEgressFoldHoldsOnUnclassifiedChargedTransfer(t *testing.T) { |
| openOwner(t, fx, foldBucketA, foldAcctA, "shop") |
| putCURDelivery(t, fx, "exec-1", curLine("l1", "INV-1", "EUW1-S3RTC-Out-Bytes", foldBucketA, "2026-09-03T10:00:00Z", "2", "0.04", "0.08")) |
| meters := map[string]map[string]float64{} |
| - res := fx.srv.foldStoreEgress(context.Background(), foldSept, meters) |
| + res := fx.srv.foldStoreEgress(context.Background(), foldSept, meters, nil) |
| if res.Complete || res.Reason != storeEgressUnclassified || res.Holds[awsbilling.ReasonUnknownTransfer] != 1 { |
| t.Fatalf("result %+v", res) |
| } |
| @@ -232,7 +241,7 @@ func TestStoreEgressFoldWithoutAReport(t *testing.T) { |
| fx := newConsoleFixture(t) |
| ctx := context.Background() |
| now := time.Date(2026, 10, 5, 6, 0, 0, 0, time.UTC) |
| - if res := fx.srv.foldStoreEgress(ctx, foldSept, nil); res.Complete || res.HoldClose || res.Reason != storeEgressUnconfigured { |
| + if res := fx.srv.foldStoreEgress(ctx, foldSept, nil, nil); res.Complete || res.HoldClose || res.Reason != storeEgressUnconfigured { |
| t.Fatalf("unconfigured: %+v", res) |
| } |
| if err := fx.srv.statementsPass(ctx, now); err != nil { |
| @@ -244,7 +253,7 @@ func TestStoreEgressFoldWithoutAReport(t *testing.T) { |
| fx2 := newConsoleFixture(t) |
| fx2.srv.cfg.StoreEgressReportPrefix = foldReportPrefix |
| - if res := fx2.srv.foldStoreEgress(ctx, foldSept, nil); res.Complete || !res.HoldClose || res.Reason != storeEgressNoDelivery { |
| + if res := fx2.srv.foldStoreEgress(ctx, foldSept, nil, nil); res.Complete || !res.HoldClose || res.Reason != storeEgressNoDelivery { |
| t.Fatalf("no delivery: %+v", res) |
| } |
| if err := fx2.srv.statementsPass(ctx, now); err != nil { |
| @@ -331,6 +340,9 @@ func TestDurabilityFoldCountsAnAttestedDay(t *testing.T) { |
| if got := st.Meters[usage.MeterDurabilityWrites1k]; !near(got, 0.002) { |
| t.Errorf("durability_writes_1k = %v, want 0.002 (two answered tenant writes)", got) |
| } |
| + if got := st.Days["2026-09-17"][usage.MeterDurabilityWrites1k]; !near(got, 0.002) { |
| + t.Errorf("durability writes on the 17th = %v, want 0.002", got) |
| + } |
| if got := st.SourceDays[usage.SourceDurability]; got != 1 { |
| t.Errorf("durability source days = %d, want 1", got) |
| } |
| @@ -379,12 +391,12 @@ func TestStoreEgressFoldHoldsOnAnUnprovenLedger(t *testing.T) { |
| fx := newConsoleFixture(t) |
| ctx := context.Background() |
| fx.srv.cfg.StoreEgressReportPrefix = foldReportPrefix |
| - if err := objstore.OpenBucketOwner(ctx, fx.rel, foldBucketA, foldAcctA, "shop", time.Date(2026, 8, 10, 0, 0, 0, 0, time.UTC), false); err != nil { |
| + if err := objstore.OpenBucketOwner(ctx, fx.rel, foldBucketA, foldAcctA, "shop", time.Date(2026, 8, 10, 0, 0, 0, 0, time.UTC), time.Date(2026, 8, 10, 0, 0, 0, 0, time.UTC), false); err != nil { |
| t.Fatal(err) |
| } |
| putCURDelivery(t, fx, "exec-1", curLine("l1", "INV-1", "EUW1-DataTransfer-Out-Bytes", foldBucketA, "2026-09-03T10:00:00Z", "2", "0.09", "0.18")) |
| meters := map[string]map[string]float64{} |
| - res := fx.srv.foldStoreEgress(ctx, foldSept, meters) |
| + res := fx.srv.foldStoreEgress(ctx, foldSept, meters, nil) |
| if res.Complete || res.Reason != storeEgressUnowned || len(res.Unowned) != 1 { |
| t.Fatalf("result %+v", res) |
| } |
| @@ -400,7 +412,7 @@ func TestStoreEgressFoldHoldsOnAChargedCloudFrontLine(t *testing.T) { |
| fx.srv.cfg.StoreEgressReportPrefix = foldReportPrefix |
| openOwner(t, fx, foldBucketA, foldAcctA, "shop") |
| putCURDelivery(t, fx, "exec-1", curLine("l1", "INV-1", "EUW1-CloudFront-Out-Bytes", foldBucketA, "2026-09-03T10:00:00Z", "7", "0.01", "0.07")) |
| - res := fx.srv.foldStoreEgress(context.Background(), foldSept, map[string]map[string]float64{}) |
| + res := fx.srv.foldStoreEgress(context.Background(), foldSept, map[string]map[string]float64{}, nil) |
| if res.Complete || res.Reason != storeEgressUnclassified || res.Holds[awsbilling.ReasonUnexpectedCharge] != 1 { |
| t.Fatalf("result %+v", res) |
| } |
| @@ -465,10 +477,11 @@ func TestStoreEgressHeldMonthIsRevisitedAfterTheNextMonthEnds(t *testing.T) { |
| func TestDurabilityFoldHoldsBeforeABoxRegisteredItsLedger(t *testing.T) { |
| ctx := context.Background() |
| day := time.Date(2026, 9, 17, 0, 0, 0, 0, time.UTC) |
| + edgeFrom := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) |
| run := func(durabilityFrom time.Time) int { |
| fx := newConsoleFixture(t) |
| - // The box has served edge traffic since the first of the month. |
| - if _, err := usage.RegisterBoxSource(ctx, fx.rel, "box1", usage.SourceEdge, time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC)); err != nil { |
| + // The box has served edge traffic since edgeFrom. |
| + if _, err := usage.RegisterBoxSource(ctx, fx.rel, "box1", usage.SourceEdge, edgeFrom); err != nil { |
| t.Fatal(err) |
| } |
| if _, err := usage.RegisterBoxSource(ctx, fx.rel, "box1", usage.SourceDurability, durabilityFrom); err != nil { |
| @@ -489,63 +502,241 @@ func TestDurabilityFoldHoldsBeforeABoxRegisteredItsLedger(t *testing.T) { |
| if got := run(day.Add(10 * time.Hour)); got != 0 { |
| t.Errorf("durability source days = %d, want 0: the ledger registered after the box began serving", got) |
| } |
| + // A new box, on its first day: each source registers on its own first |
| + // tick, seconds apart. First-seen is write-once, so holding on that |
| + // difference would hold the box's first day for good. |
| + // The days before the box existed owe nothing, so the count is compared |
| + // with a ledger registered at the very instant the edge was. |
| + edgeFrom = day.Add(8*time.Hour + 5*time.Second) |
| + same := run(edgeFrom) |
| + if got := run(day.Add(8*time.Hour + 45*time.Second)); got != same { |
| + t.Errorf("durability source days = %d, want %d: a registration race of seconds held a new box's first day", got, same) |
| + } |
| + // An hour or more is not a race. |
| + if got := run(day.Add(9*time.Hour + 5*time.Second)); got != same-1 { |
| + t.Errorf("durability source days = %d, want %d: the ledger registered an hour after the box began serving", got, same-1) |
| + } |
| +} |
| + |
| +// reopenStatement puts a closed statement back to open, as a failed write |
| +// in the closing pass would have left it. |
| +func reopenStatement(t *testing.T, fx *consoleFixture, acct, month string) { |
| + t.Helper() |
| + ctx := context.Background() |
| + raw, etag, err := fx.rel.Get(ctx, usage.StatementKey(acct, month)) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + var st usage.Statement |
| + if err := json.Unmarshal(raw, &st); err != nil { |
| + t.Fatal(err) |
| + } |
| + st.Closed = false |
| + raw, _ = json.Marshal(st) |
| + if _, err := fx.rel.Put(ctx, usage.StatementKey(acct, month), raw, etag); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +func statementExists(t *testing.T, fx *consoleFixture, acct, month string) bool { |
| + t.Helper() |
| + _, _, err := fx.rel.Get(context.Background(), usage.StatementKey(acct, month)) |
| + if err != nil && !errors.Is(err, os.ErrNotExist) { |
| + t.Fatal(err) |
| + } |
| + return err == nil |
| +} |
| + |
| +// closedSeptember is a fixture whose September closed for the platform and |
| +// for the tenant, with the tenant a console account since August. |
| +func closedSeptember(t *testing.T) *consoleFixture { |
| + t.Helper() |
| + fx := newConsoleFixture(t) |
| + fx.srv.cfg.StoreEgressReportPrefix = foldReportPrefix |
| + if err := fx.st.CreateAccount(testAccountID(), foldAcctA, "Tenant", "owner@example.test", time.Date(2026, 8, 20, 0, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + openOwner(t, fx, foldBucketA, foldAcctA, "shop") |
| + putCURDelivery(t, fx, "exec-1", curLine("l1", "INV-1", "EUW1-DataTransfer-Out-Bytes", foldBucketA, "2026-09-03T10:00:00Z", "2", "0.09", "0.18")) |
| + if err := fx.srv.statementsPass(context.Background(), time.Date(2026, 10, 5, 6, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if !septemberStatement(t, fx, fixtureAcct).Closed || !septemberStatement(t, fx, foldAcctA).Closed { |
| + t.Fatal("fixture: September should have closed for both accounts") |
| + } |
| + return fx |
| } |
| // One account's statement failed to write in the pass that closed the |
| // month. The platform statement is closed, that account's is not. The |
| -// catch-up must see it and retry, including long after six months. |
| +// catch-up must see it and retry. |
| func TestStatementCatchUpRetriesAnAccountLeftOpen(t *testing.T) { |
| + fx := closedSeptember(t) |
| + reopenStatement(t, fx, foldAcctA, "2026-09") |
| + if err := fx.srv.statementsPass(context.Background(), time.Date(2026, 11, 5, 6, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if !septemberStatement(t, fx, foldAcctA).Closed { |
| + t.Fatal("the account left open was not retried by the next catch-up") |
| + } |
| +} |
| + |
| +// The same, found for the first time nine months later, by a console that |
| +// ran no pass in between. September is far outside the old six-month |
| +// horizon, and every month between it and now is a gap with no platform |
| +// statement. A walk that stopped at the first gap, or at six months, never |
| +// reaches it. |
| +func TestStatementCatchUpReachesAcrossGapsAndPastSixMonths(t *testing.T) { |
| + fx := closedSeptember(t) |
| + reopenStatement(t, fx, foldAcctA, "2026-09") |
| + if err := fx.srv.statementsPass(context.Background(), time.Date(2027, 7, 5, 6, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if !septemberStatement(t, fx, foldAcctA).Closed { |
| + t.Fatal("September, nine months back and behind a gap, was never reached") |
| + } |
| + // The gap months themselves were never written by any pass. They lie |
| + // inside the metered range, so they are built too. |
| + for _, month := range []string{"2026-10", "2027-01", "2027-05"} { |
| + if !statementExists(t, fx, fixtureAcct, month) { |
| + t.Fatalf("gap month %s was not built", month) |
| + } |
| + } |
| + // Nothing is invented for months before metering began. |
| + if statementExists(t, fx, fixtureAcct, "2026-08") { |
| + t.Fatal("a statement was written for a month before metering began") |
| + } |
| +} |
| + |
| +// September is held open waiting on transfer evidence. The statements |
| +// worker then stops in October and is down for all of November and |
| +// December. It comes back in January with the evidence repaired: September, |
| +// behind the November gap, closes. |
| +func TestStatementCatchUpClosesAHeldMonthBehindAGap(t *testing.T) { |
| fx := newConsoleFixture(t) |
| ctx := context.Background() |
| fx.srv.cfg.StoreEgressReportPrefix = foldReportPrefix |
| openOwner(t, fx, foldBucketA, foldAcctA, "shop") |
| - putCURDelivery(t, fx, "exec-1", curLine("l1", "INV-1", "EUW1-DataTransfer-Out-Bytes", foldBucketA, "2026-09-03T10:00:00Z", "2", "0.09", "0.18")) |
| + // No delivery yet: the month holds. |
| if err := fx.srv.statementsPass(ctx, time.Date(2026, 10, 5, 6, 0, 0, 0, time.UTC)); err != nil { |
| t.Fatal(err) |
| } |
| - if !septemberStatement(t, fx, fixtureAcct).Closed || !septemberStatement(t, fx, foldAcctA).Closed { |
| - t.Fatal("fixture: September should have closed for both accounts") |
| + if septemberStatement(t, fx, fixtureAcct).Closed { |
| + t.Fatal("fixture: September closed with no transfer report") |
| + } |
| + putCURDelivery(t, fx, "exec-1", curLine("l1", "INV-1", "EUW1-DataTransfer-Out-Bytes", foldBucketA, "2026-09-03T10:00:00Z", "2", "0.09", "0.18")) |
| + if statementExists(t, fx, fixtureAcct, "2026-11") { |
| + t.Fatal("fixture: November should be a gap") |
| } |
| - // Put the tenant's statement back to open, as a failed write in the |
| - // closing pass would have left it, and make it a console account so |
| - // the catch-up knows to look for it. |
| - if err := fx.st.CreateAccount(testAccountID(), foldAcctA, "Tenant", "owner@example.test", time.Now()); err != nil { |
| + if err := fx.srv.statementsPass(ctx, time.Date(2027, 1, 5, 6, 0, 0, 0, time.UTC)); err != nil { |
| t.Fatal(err) |
| } |
| - st := septemberStatement(t, fx, foldAcctA) |
| - st.Closed = false |
| - raw, _ := json.Marshal(st) |
| - _, etag, err := fx.rel.Get(ctx, usage.StatementKey(foldAcctA, "2026-09")) |
| + if !statementExists(t, fx, fixtureAcct, "2026-11") { |
| + t.Fatal("the November gap was not built") |
| + } |
| + a := septemberStatement(t, fx, foldAcctA) |
| + if !septemberStatement(t, fx, fixtureAcct).Closed || !a.Closed || !a.MeterComplete(usage.MeterStoreEgressGB) { |
| + t.Fatalf("September behind the November gap: closed=%v transfer days=%d", a.Closed, a.SourceDays[usage.SourceStoreEgress]) |
| + } |
| +} |
| + |
| +// An account that existed in the month and has no statement for it at all |
| +// (its write failed outright) is as open as one left unclosed. An account |
| +// created after the month ended owes it nothing and does not reopen it. |
| +func TestStatementCatchUpWritesAMissingAccountStatement(t *testing.T) { |
| + fx := closedSeptember(t) |
| + ctx := context.Background() |
| + if err := fx.rel.Delete(ctx, usage.StatementKey(foldAcctA, "2026-09")); err != nil { |
| + t.Fatal(err) |
| + } |
| + november := time.Date(2026, 11, 5, 6, 0, 0, 0, time.UTC) |
| + start := time.Date(2026, 9, 1, 0, 0, 0, 0, time.UTC) |
| + accounts, err := fx.st.Accounts() |
| if err != nil { |
| t.Fatal(err) |
| } |
| - if _, err := fx.rel.Put(ctx, usage.StatementKey(foldAcctA, "2026-09"), raw, etag); err != nil { |
| + if got := fx.srv.statementMonthState(ctx, "2026-09", start.AddDate(0, 1, 0), accounts); got != statementMonthOpen { |
| + t.Fatalf("state with a missing account statement = %v, want open", got) |
| + } |
| + if err := fx.srv.statementsPass(ctx, november); err != nil { |
| t.Fatal(err) |
| } |
| - // Eight months on: far past the old six-month horizon, and every month |
| - // in between has a platform statement to walk back through. |
| - for m := 0; m < 8; m++ { |
| - at := time.Date(2026, time.Month(11+m), 5, 6, 0, 0, 0, time.UTC) |
| - if err := fx.srv.statementsPass(ctx, at); err != nil { |
| - t.Fatal(err) |
| - } |
| - if m == 0 { |
| - if !septemberStatement(t, fx, foldAcctA).Closed { |
| - t.Fatal("the account left open was not retried by the next catch-up") |
| + if !statementExists(t, fx, foldAcctA, "2026-09") || !septemberStatement(t, fx, foldAcctA).Closed { |
| + t.Fatal("the missing statement was not written and closed") |
| + } |
| + if got := fx.srv.statementMonthState(ctx, "2026-09", start.AddDate(0, 1, 0), accounts); got != statementMonthSettled { |
| + t.Fatalf("state afterwards = %v, want settled", got) |
| + } |
| + // An account created in October has no September statement and owes |
| + // none. |
| + if err := fx.st.CreateAccount(testAccountID(), "bfg", "Later", "later@example.test", time.Date(2026, 10, 9, 0, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + accounts, _ = fx.st.Accounts() |
| + if got := fx.srv.statementMonthState(ctx, "2026-09", start.AddDate(0, 1, 0), accounts); got != statementMonthSettled { |
| + t.Fatalf("a later account reopened the month: %v", got) |
| + } |
| +} |
| + |
| +// A statement that cannot be parsed is not "no statement". A corrupt |
| +// October must not end the walk and hide an open September behind it. |
| +func TestStatementCatchUpWalksPastAnUnreadableMonth(t *testing.T) { |
| + fx := closedSeptember(t) |
| + ctx := context.Background() |
| + reopenStatement(t, fx, foldAcctA, "2026-09") |
| + _, etag, err := fx.rel.Get(ctx, usage.StatementKey(fixtureAcct, "2026-10")) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := fx.rel.Put(ctx, usage.StatementKey(fixtureAcct, "2026-10"), []byte("{not json"), etag); err != nil { |
| + t.Fatal(err) |
| + } |
| + accounts, _ := fx.st.Accounts() |
| + if got := fx.srv.statementMonthState(ctx, "2026-10", time.Date(2026, 11, 1, 0, 0, 0, 0, time.UTC), accounts); got != statementMonthOpen { |
| + t.Fatalf("an unreadable platform statement = %v, want open (never absent)", got) |
| + } |
| + if err := fx.srv.statementsPass(ctx, time.Date(2026, 12, 5, 6, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if !septemberStatement(t, fx, foldAcctA).Closed { |
| + t.Fatal("an unreadable October hid the open September behind it") |
| + } |
| +} |
| + |
| +// Every billed meter's days sum to its monthly total, whatever source it |
| +// came from. A partly covered month is billed from the days, so they must |
| +// account for the month. |
| +func TestStatementDaysSumToTheMonthlyMeters(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + putStoreJSON(t, fx.rel, metrics.DayKey("box1", "2026-09-02"), watchedMetricsDay("box1", "2026-09-02")) |
| + putStoreJSON(t, fx.rel, metrics.DayKey("box1", "2026-09-09"), watchedMetricsDay("box1", "2026-09-09")) |
| + putStoreJSON(t, fx.rel, usage.EdgeDayKey("box1", "2026-09-02"), watchedEdgeDay("box1", "2026-09-02")) |
| + putStoreJSON(t, fx.rel, usage.EdgeDayKey("box1", "2026-09-11"), watchedEdgeDay("box1", "2026-09-11")) |
| + for _, date := range []string{"2026-09-05", "2026-09-06"} { |
| + putStoreJSON(t, fx.rel, usage.StorageDayKey(date), usage.StorageDay{V: 1, Date: date, |
| + Apps: map[string]usage.AppStorage{fixtureAcct + "/" + fixtureApp: {BucketBytes: 3_000_000_000}}}) |
| + } |
| + if err := fx.srv.statementsPass(ctx, time.Date(2026, 9, 20, 6, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + seen := 0 |
| + for _, acct := range []string{fixtureAcct} { |
| + st := septemberStatement(t, fx, acct) |
| + for _, meter := range monthlyBillingMeters { |
| + sum := 0.0 |
| + for _, mm := range st.Days { |
| + sum += mm[meter] |
| + } |
| + if !near(sum, st.Meters[meter]) { |
| + t.Errorf("%s %s: days sum to %v, the month says %v", acct, meter, sum, st.Meters[meter]) |
| } |
| - // Open it again: the later passes must still find it. |
| - st := septemberStatement(t, fx, foldAcctA) |
| - st.Closed = false |
| - raw, _ := json.Marshal(st) |
| - _, etag, _ := fx.rel.Get(ctx, usage.StatementKey(foldAcctA, "2026-09")) |
| - if _, err := fx.rel.Put(ctx, usage.StatementKey(foldAcctA, "2026-09"), raw, etag); err != nil { |
| - t.Fatal(err) |
| + if st.Meters[meter] != 0 { |
| + seen++ |
| } |
| - // Hold it for now by withdrawing the delivery's owner proof is |
| - // not needed: later passes retry it on their own. |
| } |
| } |
| - if !septemberStatement(t, fx, foldAcctA).Closed { |
| - t.Fatal("September was forgotten once it fell out of the old six-month window") |
| + if seen == 0 { |
| + t.Fatal("fixture recorded no billed meter at all; the invariant was not exercised") |
| } |
| } |
diff --git a/internal/console/server.go b/internal/console/server.go| index 096bc9dc8..c41a0dbf4 100644 |
| --- a/internal/console/server.go |
| +++ b/internal/console/server.go |
| @@ -693,6 +693,10 @@ type Server struct { |
| billing *billingRuntime |
| cfg Config |
| store *Store |
| + // statementClosed remembers statement keys seen closed, so the |
| + // statements catch-up does not re-read them every pass. See |
| + // statementMonthState. |
| + statementClosed sync.Map |
| // claims folds the deployment's domain claims for the read paths — the |
| // Settings render and the domains list door — reading only the claims |
| // whose bucket stamp moved since the last fold (#173). Process-lifetime |
diff --git a/internal/console/storeegressfold.go b/internal/console/storeegressfold.go| index bbaeaf89c..909f1f4a7 100644 |
| --- a/internal/console/storeegressfold.go |
| +++ b/internal/console/storeegressfold.go |
| @@ -74,8 +74,9 @@ type storeEgressResult struct { |
| // foldStoreEgress reads the month's Cost and Usage Report delivery and |
| // folds each account's charged object-store transfer into meters as |
| -// usage.MeterStoreEgressGB. period is the first instant of a UTC month. |
| -func (s *Server) foldStoreEgress(ctx context.Context, period time.Time, meters map[string]map[string]float64) storeEgressResult { |
| +// usage.MeterStoreEgressGB, and by UTC day into days when it is not nil. |
| +// period is the first instant of a UTC month. |
| +func (s *Server) foldStoreEgress(ctx context.Context, period time.Time, meters map[string]map[string]float64, days *usageSweep) storeEgressResult { |
| prefix := s.cfg.StoreEgressReportPrefix |
| if prefix == "" { |
| return storeEgressResult{Reason: storeEgressUnconfigured} |
| @@ -131,6 +132,8 @@ func (s *Server) foldStoreEgress(ctx context.Context, period time.Time, meters m |
| unowned := map[string]bool{} |
| ownerUnread := false |
| perAccount := map[string]int64{} |
| + type accountDay struct{ acct, date string } |
| + perAccountDay := map[accountDay]int64{} |
| // Deterministic order, so logs and any overflow are reproducible. |
| keys := make([]bucketHour, 0, len(charged)) |
| for k := range charged { |
| @@ -167,6 +170,10 @@ func (s *Server) foldStoreEgress(ctx context.Context, period time.Time, meters m |
| continue |
| } |
| perAccount[acct] += n |
| + perAccountDay[accountDay{acct, usage.DayOf(k.hour)}] += n |
| + } |
| + for k, n := range perAccountDay { |
| + days.addDay(k.acct, k.date, usage.MeterStoreEgressGB, float64(n)/1e9) |
| } |
| for acct, n := range perAccount { |
| mm := meters[acct] |
diff --git a/internal/host/durability_test.go b/internal/host/durability_test.go| index 173106c02..50acc1bbf 100644 |
| --- a/internal/host/durability_test.go |
| +++ b/internal/host/durability_test.go |
| @@ -32,6 +32,12 @@ func newTestDurabilityPublisher(t *testing.T) (*durabilityPublisher, string, tim |
| t.Error(err) |
| } |
| } |
| + // An escape filed after the stop voids the clean stop from a |
| + // background goroutine that writes into the ledger directory. Wait |
| + // for it, or it races the removal of the test's temp dir. |
| + if err := p.ledger.WaitLate(ctx); err != nil { |
| + t.Error(err) |
| + } |
| }) |
| return p, dir, at |
| } |
diff --git a/internal/objstore/memprovisioner.go b/internal/objstore/memprovisioner.go| index 20f4e99ef..4da037136 100644 |
| --- a/internal/objstore/memprovisioner.go |
| +++ b/internal/objstore/memprovisioner.go |
| @@ -3,6 +3,8 @@ package objstore |
| import ( |
| "context" |
| "fmt" |
| + "sort" |
| + "strings" |
| "sync" |
| ) |
| @@ -206,3 +208,23 @@ func (m *MemProvisioner) BucketExists(_ context.Context, name string) (bool, err |
| _, ok := m.Buckets[name] |
| return ok, nil |
| } |
| + |
| +// BucketGrantees lists the users whose policy names the bucket |
| +// (GranteeProber), sorted. |
| +func (m *MemProvisioner) BucketGrantees(_ context.Context, name string) ([]string, error) { |
| + m.mu.Lock() |
| + defer m.mu.Unlock() |
| + if err := m.FailNext["BucketGrantees"]; err != nil { |
| + delete(m.FailNext, "BucketGrantees") |
| + return nil, err |
| + } |
| + arn := "arn:aws:s3:::" + name |
| + var out []string |
| + for user, u := range m.Users { |
| + if strings.Contains(u.Policy, `"`+arn+`"`) || strings.Contains(u.Policy, `"`+arn+`/`) { |
| + out = append(out, user) |
| + } |
| + } |
| + sort.Strings(out) |
| + return out, nil |
| +} |
diff --git a/internal/objstore/ownership.go b/internal/objstore/ownership.go| index 211fdf503..2a8cf309a 100644 |
| --- a/internal/objstore/ownership.go |
| +++ b/internal/objstore/ownership.go |
| @@ -245,10 +245,12 @@ func writeBucketOwnership(ctx context.Context, s release.Store, bucket string, c |
| // the outcome a person needs to see. |
| // |
| // minted says this grant creates the bucket and no credential for it exists |
| -// yet. Only then, and only when this call creates the ledger, is the ledger |
| -// marked complete from `from` (BucketOwnership.CompleteSince). Every other |
| -// ledger records what is known and resolves no owner. |
| -func OpenBucketOwner(ctx context.Context, s release.Store, bucket, acct, app string, from time.Time, minted bool) error { |
| +// yet, as observed at `observed`. Only then, and only when this call creates |
| +// the ledger, is the ledger marked complete (BucketOwnership.CompleteSince), |
| +// and from the instant of the observation, not from the interval's start: a |
| +// grant block can be older than the run that probed, and the ledger vouches |
| +// for nothing before it looked. |
| +func OpenBucketOwner(ctx context.Context, s release.Store, bucket, acct, app string, from, observed time.Time, minted bool) error { |
| if !release.ValidAccount(acct) || !release.ValidApp(app) { |
| return fmt.Errorf("objstore: open bucket owner: invalid owner %q/%q", acct, app) |
| } |
| @@ -256,11 +258,19 @@ func OpenBucketOwner(ctx context.Context, s release.Store, bucket, acct, app str |
| if err := validOwnershipTime(from); err != nil { |
| return fmt.Errorf("objstore: open bucket owner %s: %v", bucket, err) |
| } |
| + if minted { |
| + if err := validOwnershipTime(observed.UTC()); err != nil { |
| + return fmt.Errorf("objstore: open bucket owner %s: observed: %v", bucket, err) |
| + } |
| + } |
| return writeBucketOwnership(ctx, s, bucket, func(b *BucketOwnership) (*BucketOwnership, error) { |
| if b == nil { |
| b = &BucketOwnership{V: 1, Bucket: bucket} |
| if minted { |
| - since := from |
| + since := observed.UTC() |
| + if since.Before(from) { |
| + since = from |
| + } |
| b.CompleteSince = &since |
| } |
| } |
| @@ -326,6 +336,9 @@ func (b *BucketOwnership) OwnerAt(hour time.Time) (acct, app string, ok bool) { |
| } |
| from := hour.UTC().Truncate(time.Hour) |
| to := from.Add(time.Hour) |
| + // The hour the ledger began vouching in resolves too: it was created |
| + // while the bucket did not exist and nobody held the name, so nothing |
| + // could have left a bucket of that name earlier in the hour. |
| if !b.CompleteSince.Before(to) { |
| return "", "", false |
| } |
diff --git a/internal/objstore/ownership_test.go b/internal/objstore/ownership_test.go| index 5354df31f..78909ed3c 100644 |
| --- a/internal/objstore/ownership_test.go |
| +++ b/internal/objstore/ownership_test.go |
| @@ -29,12 +29,12 @@ func mustLedger(t *testing.T, s release.Store, bucket string) *BucketOwnership { |
| func TestOpenBucketOwnerIsIdempotentAndNeverMovesFrom(t *testing.T) { |
| ctx := context.Background() |
| s := release.NewDirStore(t.TempDir()) |
| - if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0, true); err != nil { |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0, t0, true); err != nil { |
| t.Fatal(err) |
| } |
| // Earlier and later opens for the same owner change nothing. |
| for _, at := range []time.Time{t0.Add(-48 * time.Hour), t0.Add(48 * time.Hour)} { |
| - if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", at, true); err != nil { |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", at, at, true); err != nil { |
| t.Fatal(err) |
| } |
| } |
| @@ -54,7 +54,7 @@ func TestCloseBucketOwner(t *testing.T) { |
| if b, _, _ := ReadBucketOwnership(ctx, s, ledgerBucket); b != nil { |
| t.Fatal("closing an absent ledger must not create one") |
| } |
| - if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0, true); err != nil { |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0, t0, true); err != nil { |
| t.Fatal(err) |
| } |
| end := t0.Add(5 * time.Hour) |
| @@ -72,7 +72,7 @@ func TestCloseBucketOwner(t *testing.T) { |
| } |
| // Reopening appends; the closed interval stands. |
| again := t0.Add(72 * time.Hour) |
| - if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", again, true); err != nil { |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", again, again, true); err != nil { |
| t.Fatal(err) |
| } |
| b = mustLedger(t, s, ledgerBucket) |
| @@ -84,7 +84,7 @@ func TestCloseBucketOwner(t *testing.T) { |
| func TestCloseBeforeFromClampsToEmpty(t *testing.T) { |
| ctx := context.Background() |
| s := release.NewDirStore(t.TempDir()) |
| - if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0, true); err != nil { |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0, t0, true); err != nil { |
| t.Fatal(err) |
| } |
| if err := CloseBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0.Add(-time.Hour)); err != nil { |
| @@ -142,14 +142,14 @@ func TestOwnerAt(t *testing.T) { |
| func TestOwnerAtRefusesSharedHour(t *testing.T) { |
| ctx := context.Background() |
| s := release.NewDirStore(t.TempDir()) |
| - if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0, true); err != nil { |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0, t0, true); err != nil { |
| t.Fatal(err) |
| } |
| if err := CloseBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0.Add(time.Hour)); err != nil { |
| t.Fatal(err) |
| } |
| // Another app picks the bucket up in the same hour the first let go. |
| - if err := OpenBucketOwner(ctx, s, ledgerBucket, "bac", "other", t0.Add(90*time.Minute), true); err != nil { |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bac", "other", t0.Add(90*time.Minute), t0.Add(90*time.Minute), true); err != nil { |
| t.Fatal(err) |
| } |
| owners := BucketOwners{ledgerBucket: mustLedger(t, s, ledgerBucket)} |
| @@ -167,7 +167,7 @@ func TestOwnerAtRefusesSharedHour(t *testing.T) { |
| } |
| // Two apps opened concurrently on one adopted bucket: both stand, and |
| // every shared hour is unresolved. |
| - if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0.Add(3*time.Hour), true); err != nil { |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0.Add(3*time.Hour), t0.Add(3*time.Hour), true); err != nil { |
| t.Fatal(err) |
| } |
| b := mustLedger(t, s, ledgerBucket) |
| @@ -236,7 +236,7 @@ func TestReadBucketOwnershipRefusesBadLedgers(t *testing.T) { |
| } |
| // A writer must refuse to build on a ledger it cannot read, never |
| // overwrite it. |
| - if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0, true); err == nil { |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0, t0, true); err == nil { |
| t.Errorf("%s: open wrote over an unreadable ledger", name) |
| } |
| } |
| @@ -255,7 +255,7 @@ type racingOwnershipStore struct { |
| func (r *racingOwnershipStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| if key == BucketOwnershipKey(ledgerBucket) && !r.raced { |
| r.raced = true |
| - if err := OpenBucketOwner(ctx, r.Store, ledgerBucket, "bac", "other", t0.Add(-time.Hour), true); err != nil { |
| + if err := OpenBucketOwner(ctx, r.Store, ledgerBucket, "bac", "other", t0.Add(-time.Hour), t0.Add(-time.Hour), true); err != nil { |
| return "", err |
| } |
| } |
| @@ -265,7 +265,7 @@ func (r *racingOwnershipStore) Put(ctx context.Context, key string, body []byte, |
| func TestOpenBucketOwnerRetriesLostCAS(t *testing.T) { |
| ctx := context.Background() |
| s := &racingOwnershipStore{Store: release.NewDirStore(t.TempDir())} |
| - if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0, true); err != nil { |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0, t0, true); err != nil { |
| t.Fatal(err) |
| } |
| b := mustLedger(t, s, ledgerBucket) |
| @@ -411,11 +411,11 @@ func TestAdoptedBucketResolvesNoOwner(t *testing.T) { |
| func TestCompletenessCannotBeClaimedLater(t *testing.T) { |
| ctx := context.Background() |
| s := release.NewDirStore(t.TempDir()) |
| - if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0, false); err != nil { |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0, t0, false); err != nil { |
| t.Fatal(err) |
| } |
| // A minted=true call on an existing ledger changes nothing. |
| - if err := OpenBucketOwner(ctx, s, ledgerBucket, "bac", "other", t0.Add(time.Hour), true); err != nil { |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bac", "other", t0.Add(time.Hour), t0.Add(time.Hour), true); err != nil { |
| t.Fatal(err) |
| } |
| if b := mustLedger(t, s, ledgerBucket); b.CompleteSince != nil { |
| @@ -650,3 +650,135 @@ func TestPartialBackfillOfASharedBucketResolvesNoOwner(t *testing.T) { |
| t.Fatalf("the re-granted app %q was made owner of a bucket its mate can still reach", acct) |
| } |
| } |
| + |
| +// An absent bucket proves nothing about credentials. Before ledgers |
| +// existed, app B adopted the name app A would mint. B's teardown deleted |
| +// the bucket and then failed to delete B's user, so B's key still names the |
| +// bucket. When A is granted, the bucket does not exist, and A's ledger must |
| +// still not claim to name everyone: B would reach the new bucket at once. |
| +func TestGrantClaimsNothingWhileAnotherPrincipalNamesTheBucket(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + _, pub := testSealKeypair(t) |
| + p := NewMemProvisioner("eu-west-1") |
| + declare(t, s, "bac", "other", "OTHER_S3") |
| + if _, err := Grant(ctx, s, p, testDeployment(), pub, "bac", "other", "op", GrantSourceOperator, ledgerBucket, t0); err != nil { |
| + t.Fatal(err) |
| + } |
| + // As it was before ledgers: B is provisioned and nothing is recorded. |
| + if err := s.Delete(ctx, BucketOwnershipKey(ledgerBucket)); err != nil { |
| + t.Fatal(err) |
| + } |
| + p.FailNext = map[string]error{"DeleteUser": errors.New("iam had a bad moment")} |
| + if err := Teardown(ctx, s, p, nil, "bac", "other"); err == nil { |
| + t.Fatal("the injected DeleteUser failure did not stop the teardown") |
| + } |
| + p.FailNext = nil |
| + if _, exists := p.Buckets[ledgerBucket]; exists { |
| + t.Fatal("fixture: the bucket should be gone") |
| + } |
| + // The teardown recorded the legacy store before destroying anything: |
| + // an incomplete ledger whose interval for B is still open, because B's |
| + // key is still alive. |
| + legacy := mustLedger(t, s, ledgerBucket) |
| + if legacy.CompleteSince != nil || len(legacy.Intervals) != 1 || legacy.Intervals[0].Account != "bac" || legacy.Intervals[0].Until != nil { |
| + t.Fatalf("the failed teardown left no open, incomplete record of the legacy store: %+v", legacy) |
| + } |
| + // An older build's teardown left nothing at all. Take the record away |
| + // again: the grant below must hold on the principal inventory alone. |
| + if err := s.Delete(ctx, BucketOwnershipKey(ledgerBucket)); err != nil { |
| + t.Fatal(err) |
| + } |
| + declare(t, s, "bab", "titogo", "TITO_S3") |
| + if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", t0.Add(72*time.Hour)); err != nil { |
| + t.Fatal(err) |
| + } |
| + b := mustLedger(t, s, ledgerBucket) |
| + if b.CompleteSince != nil { |
| + t.Fatalf("claimed completeness while another principal's policy names the bucket: %v", b.CompleteSince) |
| + } |
| + if acct, _, ok := b.OwnerAt(t0.Add(96 * time.Hour)); ok { |
| + t.Fatalf("resolved %q for a bucket a surviving key can reach", acct) |
| + } |
| +} |
| + |
| +// The same failed teardown of a legacy store, with the record it now |
| +// leaves behind and a provisioner that cannot list principals (AWS today). |
| +// The new owner's grant joins a ledger that claims nothing, and while the |
| +// old key lives the two share every hour: nothing resolves. |
| +func TestFailedLegacyTeardownLeavesARecordTheNextGrantInherits(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + _, pub := testSealKeypair(t) |
| + mem := NewMemProvisioner("eu-west-1") |
| + p := noInventory{mem} |
| + declare(t, s, "bac", "other", "OTHER_S3") |
| + if _, err := Grant(ctx, s, p, testDeployment(), pub, "bac", "other", "op", GrantSourceOperator, ledgerBucket, t0); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := s.Delete(ctx, BucketOwnershipKey(ledgerBucket)); err != nil { |
| + t.Fatal(err) |
| + } |
| + mem.FailNext = map[string]error{"DeleteUser": errors.New("iam had a bad moment")} |
| + if err := Teardown(ctx, s, p, nil, "bac", "other"); err == nil { |
| + t.Fatal("the injected DeleteUser failure did not stop the teardown") |
| + } |
| + mem.FailNext = nil |
| + declare(t, s, "bab", "titogo", "TITO_S3") |
| + if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", t0.Add(72*time.Hour)); err != nil { |
| + t.Fatal(err) |
| + } |
| + b := mustLedger(t, s, ledgerBucket) |
| + if b.CompleteSince != nil || len(b.Intervals) != 2 { |
| + t.Fatalf("ledger after the reuse: %+v", b) |
| + } |
| + if acct, _, ok := b.OwnerAt(t0.Add(96 * time.Hour)); ok { |
| + t.Fatalf("resolved %q for a bucket a surviving key can reach", acct) |
| + } |
| +} |
| + |
| +// A provisioner that cannot list principals claims nothing. This is the |
| +// AWS provisioner today. |
| +type noInventory struct{ Provisioner } |
| + |
| +func TestGrantClaimsNothingWithoutAPrincipalInventory(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + _, pub := testSealKeypair(t) |
| + declare(t, s, "bab", "titogo", "TITO_S3") |
| + if _, err := Grant(ctx, s, noInventory{NewMemProvisioner("eu-west-1")}, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", t0); err != nil { |
| + t.Fatal(err) |
| + } |
| + if b := mustLedger(t, s, ledgerBucket); b.CompleteSince != nil { |
| + t.Fatalf("claimed completeness with no way to list principals: %v", b.CompleteSince) |
| + } |
| + var _ GranteeProber = NewMemProvisioner("eu-west-1") |
| + if _, ok := Provisioner(&awsProvisioner{}).(GranteeProber); ok { |
| + t.Fatal("the AWS provisioner gained a principal inventory: review the completeness claim and its IAM grant before relying on it") |
| + } |
| +} |
| + |
| +// The ledger vouches from when it looked, not from when the grant block was |
| +// written. A grant that crashed after writing its block and is re-run two |
| +// days later claims completeness from the re-run. |
| +func TestCompletenessDatesFromTheObservation(t *testing.T) { |
| + ctx := context.Background() |
| + s := &failingStore{Store: release.NewDirStore(t.TempDir()), failKeyOnce: BucketOwnershipKey(ledgerBucket)} |
| + _, pub := testSealKeypair(t) |
| + p := NewMemProvisioner("eu-west-1") |
| + declare(t, s, "bab", "titogo", "TITO_S3") |
| + if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", t0); err == nil { |
| + t.Fatal("the injected ledger failure did not stop the grant") |
| + } |
| + rerun := t0.Add(48 * time.Hour) |
| + if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", rerun); err != nil { |
| + t.Fatal(err) |
| + } |
| + b := mustLedger(t, s, ledgerBucket) |
| + if b.CompleteSince == nil || !b.CompleteSince.Equal(rerun) || !b.Intervals[0].From.Equal(t0) { |
| + t.Fatalf("complete_since %v from %v, want the re-run instant and the grant block's time", b.CompleteSince, b.Intervals[0].From) |
| + } |
| + if _, _, ok := b.OwnerAt(t0.Add(24 * time.Hour)); ok { |
| + t.Fatal("resolved an owner for an hour before the ledger looked") |
| + } |
| +} |
diff --git a/internal/objstore/provisioner.go b/internal/objstore/provisioner.go| index 8f2e7293b..5580fc0f0 100644 |
| --- a/internal/objstore/provisioner.go |
| +++ b/internal/objstore/provisioner.go |
| @@ -41,3 +41,19 @@ type Provisioner interface { |
| type BucketProber interface { |
| BucketExists(ctx context.Context, name string) (bool, error) |
| } |
| + |
| +// GranteeProber is implemented by a Provisioner that can list every |
| +// principal whose policy names a bucket, whether or not the bucket exists. |
| +// An absent bucket proves nothing about credentials: a key scoped to a |
| +// bucket name outlives the bucket (a teardown that deleted the bucket and |
| +// then failed to delete the user), and reaches a bucket of that name the |
| +// moment it is created again. Only an inventory of principals can show |
| +// that nobody is waiting on a name. |
| +// |
| +// The AWS provisioner does not implement it yet: it needs to list the store |
| +// users and read their policies, which the console's role is not granted |
| +// today. Until it does, no bucket's ownership ledger claims completeness on |
| +// AWS, and store transfer holds. |
| +type GranteeProber interface { |
| + BucketGrantees(ctx context.Context, name string) ([]string, error) |
| +} |
diff --git a/internal/objstore/walk.go b/internal/objstore/walk.go| index 5342f83e7..439a42689 100644 |
| --- a/internal/objstore/walk.go |
| +++ b/internal/objstore/walk.go |
| @@ -186,22 +186,28 @@ func Grant(ctx context.Context, s release.Store, p Provisioner, d Deployment, |
| // |
| // - the grant block says the grant minted its bucket (GrantBlock.Minted, |
| // a positive record; an adopting grant or an older block reads false); |
| - // - nothing has been delivered yet (no Provisioned block); |
| - // - the bucket does not exist yet. A bucket that is already there, under |
| - // any owner, may already be reachable by someone the ledger never saw. |
| + // - nothing has been delivered yet for this app (no Provisioned block); |
| + // - the bucket does not exist yet (BucketProber); |
| + // - no principal anywhere has a policy naming the bucket |
| + // (GranteeProber). Absence of the bucket alone proves nothing: a key |
| + // left behind by a store that once held the name, torn down before |
| + // ledgers existed and never revoked, would reach the new bucket the |
| + // moment it is created. |
| // |
| - // A provisioner that cannot say whether the bucket exists, or a probe |
| - // that fails, claims nothing. A ledger that already exists is left as it |
| - // is whatever this computes. |
| + // A provisioner that cannot answer either question, or a probe that |
| + // fails, claims nothing. A ledger that already exists is left as it is |
| + // whatever this computes. |
| minted := false |
| if rec.Grant.Minted && rec.Provisioned == nil { |
| - if prober, ok := p.(BucketProber); ok { |
| - if exists, perr := prober.BucketExists(ctx, bucket); perr == nil && !exists { |
| - minted = true |
| - } |
| + prober, canProbe := p.(BucketProber) |
| + inventory, canList := p.(GranteeProber) |
| + if canProbe && canList { |
| + exists, perr := prober.BucketExists(ctx, bucket) |
| + grantees, gerr := inventory.BucketGrantees(ctx, bucket) |
| + minted = perr == nil && gerr == nil && !exists && len(grantees) == 0 |
| } |
| } |
| - if err := OpenBucketOwner(ctx, s, bucket, acct, app, grantOwnershipFrom(rec.Grant, now), minted); err != nil { |
| + if err := OpenBucketOwner(ctx, s, bucket, acct, app, grantOwnershipFrom(rec.Grant, now), now, minted); err != nil { |
| return nil, fmt.Errorf("objstore: record ownership of %s: %w", bucket, err) |
| } |
| // 2. the principal and its policy. EnsureUser runs UNCONDITIONALLY, before |
| @@ -554,6 +560,26 @@ func Teardown(ctx context.Context, s release.Store, p Provisioner, sc Scanner, a |
| // and validates a name before writing the block), so there is nothing |
| // nameable to destroy — but the deterministic user might exist, and the |
| // delivered vars might too, so only the bucket steps are skipped. |
| + // A store provisioned before ownership ledgers existed has none. Record |
| + // it, incomplete, BEFORE anything is destroyed: if this teardown deletes |
| + // the bucket and then fails to delete the user, the surviving key still |
| + // names the bucket, and the open interval written here is the only |
| + // record that it does. Anyone granted the name afterwards finds a |
| + // ledger that claims nothing and shares its hours with this app. Only |
| + // when no ledger exists at all: an existing one already says what it |
| + // knows, and a re-run after the close below must not reopen it. |
| + if bucket != "" && ValidBucketName(bucket) { |
| + existing, _, lerr := ReadBucketOwnership(ctx, s, bucket) |
| + if lerr != nil { |
| + return fmt.Errorf("objstore: read ownership of %s: %w", bucket, lerr) |
| + } |
| + if existing == nil { |
| + now := ownershipNow() |
| + if err := OpenBucketOwner(ctx, s, bucket, acct, app, grantOwnershipFrom(rec.Grant, now), now, false); err != nil { |
| + return fmt.Errorf("objstore: record ownership of %s: %w", bucket, err) |
| + } |
| + } |
| + } |
| if bucket != "" { |
| // Empty before delete: a real backend refuses to delete a bucket with |
| // objects in it, and EmptyBucket is paginated and resumable for exactly |
diff --git a/internal/plans/month.go b/internal/plans/month.go| index 75d1f3a8b..3807fe81d 100644 |
| --- a/internal/plans/month.go |
| +++ b/internal/plans/month.go |
| @@ -21,14 +21,47 @@ type Assignment struct { |
| PaidThrough time.Time `json:"paid_through"` |
| } |
| +// CoveredInterval is one span of a month during which a paid plan was in |
| +// force: an active, paid assignment, up to what it had been paid through. |
| +type CoveredInterval struct { |
| + From time.Time `json:"from"` |
| + To time.Time `json:"to"` |
| +} |
| + |
| type MonthAllowance struct { |
| Month string `json:"month"` |
| TermsVersion string `json:"terms_version"` |
| House bool `json:"house"` |
| Resources Resources `json:"resources"` |
| Assignments []Assignment `json:"assignments"` |
| + // Partial is true when paid plans covered only part of the month: it |
| + // began before the first paid assignment, or a subscription ended, was |
| + // not active, or had not been paid through, for part of it. Covered |
| + // then lists the paid spans, in order, and Resources is the allowance |
| + // for those spans alone. Usage outside them is not this month's to |
| + // bill, and the caller must count only usage it can place inside them. |
| + // Both are absent for a month covered from its first instant to its |
| + // last, which reads exactly as it always has. |
| + Partial bool `json:"partial,omitempty"` |
| + Covered []CoveredInterval `json:"covered,omitempty"` |
| } |
| +// AllowanceForMonth resolves the pooled allowance a month's usage is set |
| +// against, from the account's effective-dated assignment history. |
| +// |
| +// Each assignment is in force from its effective instant until the next |
| +// one's. It is PAID for that span, up to its PaidThrough, when it is an |
| +// active assignment of a paid plan. The allowance is each paid span's |
| +// pooled allowance weighted by the span's share of the month, rounded up |
| +// for the customer. |
| +// |
| +// A month the paid spans do not cover end to end is Partial, never a |
| +// refusal: a first month that starts mid-month, a month a subscription |
| +// ends in, a lapse, a span not paid through. What is refused |
| +// (ErrAssignmentHistory) is a history that cannot be read at all (empty, |
| +// unordered, zero times or revisions), a month with no paid span, and paid |
| +// spans that disagree on the terms version or the house marker, whose |
| +// usage could not be priced on one basis. |
| func AllowanceForMonth(month string, history []Assignment) (*MonthAllowance, error) { |
| start, err := time.Parse("2006-01", month) |
| if err != nil || start.Format("2006-01") != month || len(history) == 0 { |
| @@ -61,12 +94,16 @@ func AllowanceForMonth(month string, history []Assignment) (*MonthAllowance, err |
| first = i |
| } |
| } |
| + out := &MonthAllowance{Month: month} |
| + from := first |
| if first < 0 { |
| - return nil, ErrAssignmentHistory |
| + // Nothing was in force when the month began. |
| + out.Partial = true |
| + from = 0 |
| } |
| - out := &MonthAllowance{Month: month, TermsVersion: assignments[first].TermsVersion, House: assignments[first].House} |
| var totals [4]big.Int |
| - for i := first; i < len(assignments); i++ { |
| + paid := false |
| + for i := from; i < len(assignments); i++ { |
| a := assignments[i] |
| if !a.EffectiveAt.Before(end) { |
| break |
| @@ -81,7 +118,20 @@ func AllowanceForMonth(month string, history []Assignment) (*MonthAllowance, err |
| if !lo.Before(hi) { |
| continue |
| } |
| - if a.TermsVersion != out.TermsVersion || a.House != out.House || a.Plan == PlanFree || a.Status != "active" || a.PaidThrough.Before(hi) { |
| + paidHi := hi |
| + if a.PaidThrough.Before(paidHi) { |
| + paidHi = a.PaidThrough |
| + } |
| + if a.Plan == PlanFree || a.Status != "active" || !lo.Before(paidHi) { |
| + out.Partial = true |
| + continue |
| + } |
| + if paidHi.Before(hi) { |
| + out.Partial = true |
| + } |
| + if !paid { |
| + out.TermsVersion, out.House, paid = a.TermsVersion, a.House, true |
| + } else if a.TermsVersion != out.TermsVersion || a.House != out.House { |
| return nil, ErrAssignmentHistory |
| } |
| quote, err := Quote(a.TermsVersion, a.Plan, a.Band, a.Quantity) |
| @@ -90,10 +140,23 @@ func AllowanceForMonth(month string, history []Assignment) (*MonthAllowance, err |
| } |
| values := []int64{quote.Allowances.AwakeSeconds, quote.Allowances.StorageByteMonths, quote.Allowances.TransferBytes, quote.Allowances.Writes} |
| for n, value := range values { |
| - weighted := new(big.Int).Mul(big.NewInt(value), big.NewInt(int64(hi.Sub(lo)))) |
| + weighted := new(big.Int).Mul(big.NewInt(value), big.NewInt(int64(paidHi.Sub(lo)))) |
| totals[n].Add(&totals[n], weighted) |
| } |
| out.Assignments = append(out.Assignments, a) |
| + // Adjacent paid spans join, so a quantity change mid-month does not |
| + // split a day between two intervals. |
| + if n := len(out.Covered); n > 0 && out.Covered[n-1].To.Equal(lo) { |
| + out.Covered[n-1].To = paidHi |
| + } else { |
| + out.Covered = append(out.Covered, CoveredInterval{From: lo, To: paidHi}) |
| + } |
| + } |
| + if !paid { |
| + return nil, ErrAssignmentHistory |
| + } |
| + if !out.Partial { |
| + out.Covered = nil |
| } |
| denominator := big.NewInt(int64(end.Sub(start))) |
| values := [4]int64{} |
diff --git a/internal/plans/month_test.go b/internal/plans/month_test.go| index bca22f555..47aa86129 100644 |
| --- a/internal/plans/month_test.go |
| +++ b/internal/plans/month_test.go |
| @@ -82,32 +82,29 @@ func TestMonthAllowanceRefusesBackdatedSupersession(t *testing.T) { |
| func TestMonthAllowanceRefusesUnpriceableHistory(t *testing.T) { |
| first := monthAssignment("2026-09-01T00:00:00Z", 1, 10) |
| second := monthAssignment("2026-09-16T00:00:00Z", 2, 20) |
| - for _, kind := range []string{"empty", "midmonth_start", "duplicate_revision", "mixed_house", "mixed_terms", "free_interval", "zero_time", "zero_revision", "canceled", "unpaid", "term_ended"} { |
| + for _, kind := range []string{"empty", "duplicate_revision", "mixed_house", "mixed_terms", "zero_time", "zero_revision", "never_paid", "only_free", "starts_after_the_month"} { |
| t.Run(kind, func(t *testing.T) { |
| history := []Assignment{first, second} |
| switch kind { |
| case "empty": |
| history = nil |
| - case "midmonth_start": |
| - history = []Assignment{second} |
| case "duplicate_revision": |
| history[1].Revision = first.Revision |
| case "mixed_house": |
| history[1].House = true |
| case "mixed_terms": |
| history[1].TermsVersion = "future" |
| - case "free_interval": |
| - history[0].Plan = PlanFree |
| case "zero_time": |
| history[0].EffectiveAt = time.Time{} |
| case "zero_revision": |
| history[0].Revision = 0 |
| - case "canceled": |
| - history[1].Status = "canceled" |
| - case "unpaid": |
| - history[1].Status = "past_due" |
| - case "term_ended": |
| - history[1].PaidThrough = history[1].EffectiveAt.Add(time.Hour) |
| + case "never_paid": |
| + // No span of the month was an active paid plan. |
| + history[0].Status, history[1].Status = "past_due", "canceled" |
| + case "only_free": |
| + history[0].Plan, history[1].Plan = PlanFree, PlanFree |
| + case "starts_after_the_month": |
| + history = []Assignment{monthAssignment("2026-10-02T00:00:00Z", 1, 10)} |
| } |
| if _, err := AllowanceForMonth("2026-09", history); !errors.Is(err, ErrAssignmentHistory) { |
| t.Fatalf("unpriceable history: %v", err) |
| @@ -116,6 +113,73 @@ func TestMonthAllowanceRefusesUnpriceableHistory(t *testing.T) { |
| } |
| } |
| +// A month the paid plan covered only partly is not refused: it yields the |
| +// paid spans and the allowance for those spans alone. September has 30 |
| +// days; Business allows 170 awake hours per member per month. |
| +func TestMonthAllowanceForAPartlyCoveredMonth(t *testing.T) { |
| + sep := func(day, hour int) time.Time { return time.Date(2026, 9, day, hour, 0, 0, 0, time.UTC) } |
| + oct1 := time.Date(2026, 10, 1, 0, 0, 0, 0, time.UTC) |
| + first := monthAssignment("2026-09-01T00:00:00Z", 1, 10) |
| + second := monthAssignment("2026-09-16T00:00:00Z", 2, 20) |
| + cases := []struct { |
| + name string |
| + history func() []Assignment |
| + covered []CoveredInterval |
| + awakeHours int64 // whole hours, for the cases that land on one |
| + awakeSecs int64 |
| + assigned int |
| + }{ |
| + {"a first month that starts mid-month", func() []Assignment { return []Assignment{second} }, |
| + []CoveredInterval{{sep(16, 0), oct1}}, 1700, 0, 1}, |
| + {"free until the sixteenth, then paid", func() []Assignment { |
| + f := first |
| + f.Plan = PlanFree |
| + return []Assignment{f, second} |
| + }, []CoveredInterval{{sep(16, 0), oct1}}, 1700, 0, 1}, |
| + {"cancelled on the sixteenth", func() []Assignment { |
| + s := second |
| + s.Status = "canceled" |
| + return []Assignment{first, s} |
| + }, []CoveredInterval{{sep(1, 0), sep(16, 0)}}, 850, 0, 1}, |
| + {"payment failed on the sixteenth", func() []Assignment { |
| + s := second |
| + s.Status = "past_due" |
| + return []Assignment{first, s} |
| + }, []CoveredInterval{{sep(1, 0), sep(16, 0)}}, 850, 0, 1}, |
| + {"paid through one hour into the second assignment", func() []Assignment { |
| + s := second |
| + s.PaidThrough = s.EffectiveAt.Add(time.Hour) |
| + return []Assignment{first, s} |
| + }, []CoveredInterval{{sep(1, 0), sep(16, 1)}}, 0, 850*3600 + 17000, 2}, |
| + } |
| + for _, c := range cases { |
| + t.Run(c.name, func(t *testing.T) { |
| + got, err := AllowanceForMonth("2026-09", c.history()) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if !got.Partial || !reflect.DeepEqual(got.Covered, c.covered) { |
| + t.Fatalf("partial=%v covered=%+v, want %+v", got.Partial, got.Covered, c.covered) |
| + } |
| + want := c.awakeSecs |
| + if want == 0 { |
| + want = c.awakeHours * 3600 |
| + } |
| + if got.Resources.AwakeSeconds != want { |
| + t.Fatalf("allowance %d awake seconds, want %d", got.Resources.AwakeSeconds, want) |
| + } |
| + if len(got.Assignments) != c.assigned { |
| + t.Fatalf("billable assignments: %+v", got.Assignments) |
| + } |
| + }) |
| + } |
| + // A month covered end to end reads exactly as it always has. |
| + whole, err := AllowanceForMonth("2026-09", []Assignment{first, second}) |
| + if err != nil || whole.Partial || whole.Covered != nil { |
| + t.Fatalf("whole month: %+v %v", whole, err) |
| + } |
| +} |
| + |
| func TestMonthAllowanceRoundsFractionalBaseUnitsForCustomer(t *testing.T) { |
| first := monthAssignment("2026-02-01T00:00:00Z", 1, 10) |
| second := monthAssignment("2026-02-01T00:00:01Z", 2, 11) |
diff --git a/internal/usage/billable_test.go b/internal/usage/billable_test.go| index 69385afac..6d1bfad22 100644 |
| --- a/internal/usage/billable_test.go |
| +++ b/internal/usage/billable_test.go |
| @@ -128,7 +128,8 @@ func TestBucketOwnershipEvidenceKeepsTheCompletenessBoundary(t *testing.T) { |
| t.Fatal(err) |
| } |
| } |
| - put(from) |
| + // Complete since before the month began. |
| + put(time.Date(2026, 8, 10, 0, 0, 0, 0, time.UTC)) |
| es, err := NewMonthEvidenceStore(raw, "2026-09") |
| if err != nil { |
| t.Fatal(err) |
| @@ -143,6 +144,13 @@ func TestBucketOwnershipEvidenceKeepsTheCompletenessBoundary(t *testing.T) { |
| if err := ev.Verify(ctx, raw); err != nil { |
| t.Fatal(err) |
| } |
| + // A boundary that moves before the month began proves the same thing |
| + // about September: the whole month, from its first instant. It is not |
| + // a change, and must not hold a month it does not touch. |
| + put(time.Date(2026, 8, 25, 0, 0, 0, 0, time.UTC)) |
| + if err := ev.Verify(ctx, raw); err != nil { |
| + t.Fatalf("a boundary moved before the month read as changed evidence: %v", err) |
| + } |
| put(time.Date(2026, 9, 20, 0, 0, 0, 0, time.UTC)) |
| if err := ev.Verify(ctx, raw); !errors.Is(err, ErrEvidenceChanged) { |
| t.Fatalf("a completeness boundary moved inside the month must read as changed evidence, got %v", err) |
diff --git a/internal/usage/statement.go b/internal/usage/statement.go| index 884b12b9f..490357a7d 100644 |
| --- a/internal/usage/statement.go |
| +++ b/internal/usage/statement.go |
| @@ -32,9 +32,16 @@ type Statement struct { |
| // THIS meter's source report every day", which is the billing one. |
| // Absent (a statement written before this field existed) reads as |
| // unknown, never as complete. |
| - SourceDays map[string]int `json:"source_days,omitempty"` |
| - DaysInMonth int `json:"days_in_month"` |
| - UpdatedAt time.Time `json:"updated_at"` |
| + SourceDays map[string]int `json:"source_days,omitempty"` |
| + // Days is the month's billed meters by UTC day ("2006-01-02" -> meter |
| + // -> value), so a month a paid plan covered only partly can be billed |
| + // on the usage that fell inside the paid days and nothing else. Each |
| + // meter's days sum to its entry in Meters. Absent on a statement |
| + // written before the field existed; such a statement can bill a whole |
| + // month and never a partial one. |
| + Days map[string]map[string]float64 `json:"days,omitempty"` |
| + DaysInMonth int `json:"days_in_month"` |
| + UpdatedAt time.Time `json:"updated_at"` |
| // Closed marks a month that will not be recomputed again: past the |
| // grace, every source has had time to land its final hours. Billing |
| // reads only closed statements, so a bill cannot be drawn from a month |