amadan
Operator evidence for a bucket's partial first hour: recorded per bucket-hour, honoured by the store-transfer fold, and nothing else is released
The hour a complete ownership ledger began vouching in does not resolve an owner, and a charged line in it holds the month. It still holds by default. An operator who has checked the provider's trail for the bucket name can now record, for that one hour, who is shown to have owned it and on what evidence (usage/bucket-evidence/<bucket>.json, append-only, with a recorded withdrawal). The fold counts the hour for the owner the ledger names and reports which hours it counted that way. The door accepts and honours an entry only for the hour the ledger began vouching in and only for the owner the ledger names. A ledger with no completeness claim, a shared hour and any other hour stay held whatever is recorded. The conditions are checked again at every fold. A month closed on evidence pins the record: a later withdrawal reads as changed evidence. Three operator doors under /api/billing/bucket-evidence (list, record, withdraw), operators only, each change audited. No CLI verb. The door's sentences are new operator-facing text and have not been reviewed. Not built here: the same door for a box's first minutes (OUTSTANDING item 4). Not gated, not reviewed.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
6 files changed,
+998
−1
internal/console/billingoperator.go+151 −0internal/console/bucketevidence_test.go+270 −0internal/console/server.go+3 −0internal/console/storeegressfold.go+26 −1internal/objstore/hourevidence.go+335 −0internal/objstore/hourevidence_test.go+213 −0
diff --git a/internal/console/billingoperator.go b/internal/console/billingoperator.go| index 37df56ab2..3eecaad2a 100644 |
| --- a/internal/console/billingoperator.go |
| +++ b/internal/console/billingoperator.go |
| @@ -7,6 +7,8 @@ import ( |
| "net/http" |
| "strings" |
| "time" |
| + |
| + "github.com/carlosframework/platform/internal/objstore" |
| ) |
| // Operator doors for billing (pricing spec step 10 and the review holds). |
| @@ -309,3 +311,152 @@ func (s *Server) handleAPIBillingHouseSet(w http.ResponseWriter, r *http.Request |
| jsonError(w, http.StatusInternalServerError, "could not change the house marker. try again") |
| } |
| } |
| + |
| +// Bucket-hour evidence (objstore/hourevidence.go). The store-transfer fold |
| +// holds a month on a charged line in the hour a bucket's ledger began |
| +// vouching in. An operator who has looked at the provider's trail for that |
| +// bucket name records what it showed here, and the fold then counts the |
| +// hour for the owner the ledger names. The door accepts nothing else: no |
| +// other hour, no other owner. |
| + |
| +type bucketEvidenceRow struct { |
| + Hour string `json:"hour"` |
| + Account string `json:"account"` |
| + App string `json:"app"` |
| + Evidence string `json:"evidence"` |
| + By string `json:"by"` |
| + At string `json:"at"` |
| + WithdrawnAt string `json:"withdrawn_at,omitempty"` |
| + WithdrawnBy string `json:"withdrawn_by,omitempty"` |
| + WithdrawnNote string `json:"withdrawn_note,omitempty"` |
| +} |
| + |
| +func (s *Server) handleAPIBillingBucketEvidenceList(w http.ResponseWriter, r *http.Request) { |
| + if _, ok := s.billingOperator(w, r); !ok { |
| + return |
| + } |
| + bucket := r.URL.Query().Get("bucket") |
| + if !objstore.ValidBucketName(bucket) { |
| + jsonError(w, http.StatusBadRequest, "a bucket is required") |
| + return |
| + } |
| + ledger, _, err := objstore.ReadBucketOwnership(r.Context(), s.cfg.Releases, bucket) |
| + if err == nil && ledger == nil { |
| + jsonError(w, http.StatusNotFound, "no ownership ledger for that bucket") |
| + return |
| + } |
| + var ev *objstore.BucketEvidence |
| + if err == nil { |
| + ev, _, err = objstore.ReadBucketEvidence(r.Context(), s.cfg.Releases, bucket) |
| + } |
| + if err != nil { |
| + log.Printf("console: api billing bucket evidence %s: %v", bucket, err) |
| + jsonError(w, http.StatusInternalServerError, "could not read the bucket's records. try again") |
| + return |
| + } |
| + out := map[string]any{"bucket": bucket, "evidence": []bucketEvidenceRow{}} |
| + if ledger.CompleteSince != nil { |
| + // The hour evidence can be recorded for, when there is one. |
| + out["complete_since"] = ledger.CompleteSince.UTC().Format(time.RFC3339) |
| + if first := ledger.CompleteSince.UTC().Truncate(time.Hour); !first.Equal(ledger.CompleteSince.UTC()) { |
| + out["held_hour"] = first.Format(time.RFC3339) |
| + } |
| + } |
| + if ev != nil { |
| + rows := make([]bucketEvidenceRow, 0, len(ev.Hours)) |
| + for _, h := range ev.Hours { |
| + row := bucketEvidenceRow{Hour: h.Hour.UTC().Format(time.RFC3339), Account: h.Account, App: h.App, Evidence: h.Evidence, By: h.By, At: h.At.UTC().Format(time.RFC3339)} |
| + if h.WithdrawnAt != nil { |
| + row.WithdrawnAt, row.WithdrawnBy, row.WithdrawnNote = h.WithdrawnAt.UTC().Format(time.RFC3339), h.WithdrawnBy, h.WithdrawnNote |
| + } |
| + rows = append(rows, row) |
| + } |
| + out["evidence"] = rows |
| + } |
| + writeJSON(w, http.StatusOK, out) |
| +} |
| + |
| +func (s *Server) handleAPIBillingBucketEvidenceRecord(w http.ResponseWriter, r *http.Request) { |
| + addr, ok := s.billingOperator(w, r) |
| + if !ok { |
| + return |
| + } |
| + var req struct { |
| + Bucket string `json:"bucket"` |
| + Hour string `json:"hour"` |
| + Account string `json:"account"` |
| + App string `json:"app"` |
| + Evidence string `json:"evidence"` |
| + } |
| + if !readJSON(w, r, &req) { |
| + return |
| + } |
| + hour, perr := time.Parse(time.RFC3339, req.Hour) |
| + acct, aerr := s.store.AccountBySqid(req.Account) |
| + if perr != nil || aerr != nil || acct == nil { |
| + jsonError(w, http.StatusBadRequest, "bucket, hour, account, app and evidence are required") |
| + return |
| + } |
| + err := objstore.RecordBucketHourEvidence(r.Context(), s.cfg.Releases, req.Bucket, hour, acct.Sqid, req.App, req.Evidence, addr, time.Now()) |
| + switch { |
| + case err == nil: |
| + s.audit(acct, req.App, addr, "billing-bucket-evidence", req.Bucket+" "+hour.UTC().Format(time.RFC3339)) |
| + writeJSON(w, http.StatusOK, map[string]any{"bucket": req.Bucket, "hour": hour.UTC().Format(time.RFC3339), "account": acct.Sqid, "app": req.App, "recorded": true}) |
| + case errors.Is(err, objstore.ErrEvidenceInvalid): |
| + jsonError(w, http.StatusBadRequest, "bucket, hour, account, app and evidence are required. the hour is the first instant of a utc hour") |
| + case errors.Is(err, objstore.ErrEvidenceNotFirstHour): |
| + jsonError(w, http.StatusConflict, "evidence is accepted only for the hour the bucket's ledger began vouching in, and only for the owner the ledger names. every other hour stays held") |
| + case errors.Is(err, objstore.ErrEvidenceExists): |
| + jsonError(w, http.StatusConflict, "that bucket-hour has a standing entry that says something else. withdraw it first") |
| + default: |
| + log.Printf("console: api billing bucket evidence record %s: %v", req.Bucket, err) |
| + jsonError(w, http.StatusInternalServerError, "could not record the evidence. try again") |
| + } |
| +} |
| + |
| +func (s *Server) handleAPIBillingBucketEvidenceWithdraw(w http.ResponseWriter, r *http.Request) { |
| + addr, ok := s.billingOperator(w, r) |
| + if !ok { |
| + return |
| + } |
| + var req struct { |
| + Bucket string `json:"bucket"` |
| + Hour string `json:"hour"` |
| + Note string `json:"note"` |
| + } |
| + if !readJSON(w, r, &req) { |
| + return |
| + } |
| + hour, perr := time.Parse(time.RFC3339, req.Hour) |
| + if perr != nil { |
| + jsonError(w, http.StatusBadRequest, "bucket, hour and a note are required") |
| + return |
| + } |
| + // Who the entry was about, for the audit trail, read before it changes. |
| + var acct *Account |
| + app := "" |
| + if ev, _, err := objstore.ReadBucketEvidence(r.Context(), s.cfg.Releases, req.Bucket); err == nil && ev != nil { |
| + for _, h := range ev.Hours { |
| + if h.WithdrawnAt == nil && h.Hour.Equal(hour) { |
| + if a, aerr := s.store.AccountBySqid(h.Account); aerr == nil { |
| + acct, app = a, h.App |
| + } |
| + } |
| + } |
| + } |
| + err := objstore.WithdrawBucketHourEvidence(r.Context(), s.cfg.Releases, req.Bucket, hour, addr, req.Note, time.Now()) |
| + switch { |
| + case err == nil: |
| + if acct != nil { |
| + s.audit(acct, app, addr, "billing-bucket-evidence-withdraw", req.Bucket+" "+hour.UTC().Format(time.RFC3339)) |
| + } |
| + writeJSON(w, http.StatusOK, map[string]any{"bucket": req.Bucket, "hour": hour.UTC().Format(time.RFC3339), "withdrawn": true}) |
| + case errors.Is(err, objstore.ErrEvidenceInvalid): |
| + jsonError(w, http.StatusBadRequest, "bucket, hour and a note are required") |
| + case errors.Is(err, objstore.ErrEvidenceNotFound): |
| + jsonError(w, http.StatusNotFound, "no standing evidence for that bucket-hour") |
| + default: |
| + log.Printf("console: api billing bucket evidence withdraw %s: %v", req.Bucket, err) |
| + jsonError(w, http.StatusInternalServerError, "could not withdraw the evidence. try again") |
| + } |
| +} |
diff --git a/internal/console/bucketevidence_test.go b/internal/console/bucketevidence_test.go| new file mode 100644 |
| index 000000000..aef150622 |
| --- /dev/null |
| +++ b/internal/console/bucketevidence_test.go |
| @@ -0,0 +1,270 @@ |
| +package console |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "net/http" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/objstore" |
| + "github.com/carlosframework/platform/internal/usage" |
| +) |
| + |
| +var ( |
| + evidenceLooked = time.Date(2026, 9, 3, 10, 20, 0, 0, time.UTC) |
| + evidenceHour = time.Date(2026, 9, 3, 10, 0, 0, 0, time.UTC) |
| +) |
| + |
| +const evidenceText = "CloudTrail eu-west-1, lookup by bucket name 09:00 to 10:20 UTC: one CreateBucket at 10:20:04, nothing earlier" |
| + |
| +// firstHourFixture: a store whose ledger began vouching at 10:20, with a |
| +// charged line in that hour and one in the next. |
| +func firstHourFixture(t *testing.T) *consoleFixture { |
| + t.Helper() |
| + fx := newConsoleFixture(t) |
| + fx.srv.cfg.StoreEgressReportPrefix = foldReportPrefix |
| + if err := objstore.OpenBucketOwner(context.Background(), fx.rel, foldBucketA, foldAcctA, "shop", evidenceLooked, evidenceLooked, true); err != nil { |
| + t.Fatal(err) |
| + } |
| + putCURDelivery(t, fx, "exec-1", |
| + curLine("l1", "INV-1", "EUW1-DataTransfer-Out-Bytes", foldBucketA, "2026-09-03T10:00:00Z", "2", "0.09", "0.18")+ |
| + curLine("l2", "INV-1", "EUW1-DataTransfer-Out-Bytes", foldBucketA, "2026-09-03T11:00:00Z", "1", "0.09", "0.09")) |
| + return fx |
| +} |
| + |
| +// The hour a ledger began vouching in holds the month by default. With an |
| +// operator's evidence recorded for that hour it counts, for the owner the |
| +// ledger names, and the fold says which hours it counted that way. With |
| +// the evidence withdrawn it holds again. |
| +func TestStoreEgressFirstHourHoldsUntilEvidenceIsRecorded(t *testing.T) { |
| + fx := firstHourFixture(t) |
| + ctx := context.Background() |
| + fold := func() (storeEgressResult, float64) { |
| + meters := map[string]map[string]float64{} |
| + res := fx.srv.foldStoreEgress(ctx, foldSept, meters, nil) |
| + return res, meters[foldAcctA][usage.MeterStoreEgressGB] |
| + } |
| + res, gb := fold() |
| + if res.Complete || res.Reason != storeEgressUnowned || len(res.Unowned) != 1 || len(res.Evidenced) != 0 { |
| + t.Fatalf("by default: %+v", res) |
| + } |
| + if !near(gb, 1) { |
| + t.Fatalf("by default the held hour was counted: %v GB, want 1 (the next hour only)", gb) |
| + } |
| + if err := objstore.RecordBucketHourEvidence(ctx, fx.rel, foldBucketA, evidenceHour, foldAcctA, "shop", evidenceText, "ops@keymail.dev", time.Date(2026, 10, 2, 9, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + res, gb = fold() |
| + if !res.Complete || len(res.Unowned) != 0 { |
| + t.Fatalf("with evidence: %+v", res) |
| + } |
| + if !near(gb, 3) { |
| + t.Fatalf("with evidence: %v GB, want 3", gb) |
| + } |
| + if len(res.Evidenced) != 1 || res.Evidenced[0] != foldBucketA+" 2026-09-03T10:00:00Z by ops@keymail.dev" { |
| + t.Fatalf("the fold did not say what it counted on evidence: %v", res.Evidenced) |
| + } |
| + if err := objstore.WithdrawBucketHourEvidence(ctx, fx.rel, foldBucketA, evidenceHour, "ops@keymail.dev", "the query covered the wrong region", time.Date(2026, 10, 2, 10, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + res, gb = fold() |
| + if res.Complete || res.Reason != storeEgressUnowned || !near(gb, 1) { |
| + t.Fatalf("after the withdrawal: %+v, %v GB", res, gb) |
| + } |
| +} |
| + |
| +// Evidence releases nothing but the first hour. A ledger that claims no |
| +// completeness holds whatever is in the evidence record, even one written |
| +// by hand in the right shape. |
| +func TestStoreEgressEvidenceReleasesNoOtherHold(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + fx.srv.cfg.StoreEgressReportPrefix = foldReportPrefix |
| + if err := objstore.OpenBucketOwner(ctx, fx.rel, foldBucketA, foldAcctA, "shop", evidenceLooked, evidenceLooked, false); err != nil { |
| + t.Fatal(err) |
| + } |
| + putCURDelivery(t, fx, "exec-1", curLine("l1", "INV-1", "EUW1-DataTransfer-Out-Bytes", foldBucketA, "2026-09-03T10:00:00Z", "2", "0.09", "0.18")) |
| + body := `{"v":1,"bucket":"` + foldBucketA + `","hours":[{"hour":"2026-09-03T10:00:00Z","account_sqid":"` + foldAcctA + `","app":"shop","evidence":"written by hand","by":"someone","at":"2026-10-02T09:00:00Z"}]}` |
| + if _, err := fx.rel.Put(ctx, objstore.BucketEvidenceKey(foldBucketA), []byte(body), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + meters := map[string]map[string]float64{} |
| + res := fx.srv.foldStoreEgress(ctx, foldSept, meters, nil) |
| + if res.Complete || res.Reason != storeEgressUnowned || len(res.Evidenced) != 0 || len(meters) != 0 { |
| + t.Fatalf("an unproven ledger was released by an evidence record: %+v %v", res, meters) |
| + } |
| +} |
| + |
| +// Evidence never chooses who pays. An entry that names anyone but the owner |
| +// the ledger names for its first hour (only a hand edit could write one) |
| +// is not honoured, and nobody is billed for the hour. |
| +func TestStoreEgressEvidenceCannotNameAnotherOwner(t *testing.T) { |
| + fx := firstHourFixture(t) |
| + ctx := context.Background() |
| + body := `{"v":1,"bucket":"` + foldBucketA + `","hours":[{"hour":"2026-09-03T10:00:00Z","account_sqid":"` + foldAcctB + `","app":"blog","evidence":"written by hand","by":"someone","at":"2026-10-02T09:00:00Z"}]}` |
| + if _, err := fx.rel.Put(ctx, objstore.BucketEvidenceKey(foldBucketA), []byte(body), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + meters := map[string]map[string]float64{} |
| + res := fx.srv.foldStoreEgress(ctx, foldSept, meters, nil) |
| + if res.Complete || res.Reason != storeEgressUnowned || len(res.Evidenced) != 0 { |
| + t.Fatalf("result %+v", res) |
| + } |
| + if _, billed := meters[foldAcctB]; billed || !near(meters[foldAcctA][usage.MeterStoreEgressGB], 1) { |
| + t.Fatalf("an entry naming another owner moved the hour: %v", meters) |
| + } |
| +} |
| + |
| +// An evidence record that cannot be read is a hold a person must look at, |
| +// not "no evidence" and not evidence. |
| +func TestStoreEgressHoldsOnUnreadableEvidence(t *testing.T) { |
| + fx := firstHourFixture(t) |
| + ctx := context.Background() |
| + if _, err := fx.rel.Put(ctx, objstore.BucketEvidenceKey(foldBucketA), []byte(`{`), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + res := fx.srv.foldStoreEgress(ctx, foldSept, map[string]map[string]float64{}, nil) |
| + if res.Complete || res.Reason != storeEgressOwnerUnread { |
| + t.Fatalf("result %+v", res) |
| + } |
| +} |
| + |
| +// A month closed on evidence keeps that evidence in what it pins: a |
| +// withdrawal after the close reads as changed evidence, so the statement |
| +// cannot be charged as it stands. |
| +func TestStatementClosedOnEvidencePinsIt(t *testing.T) { |
| + fx := firstHourFixture(t) |
| + ctx := context.Background() |
| + if err := objstore.RecordBucketHourEvidence(ctx, fx.rel, foldBucketA, evidenceHour, foldAcctA, "shop", evidenceText, "ops@keymail.dev", time.Date(2026, 10, 2, 9, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := fx.srv.statementsPass(ctx, time.Date(2026, 10, 5, 6, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + a := septemberStatement(t, fx, foldAcctA) |
| + if got := a.Meters[usage.MeterStoreEgressGB]; !near(got, 3) { |
| + t.Fatalf("store egress = %v GB, want 3", got) |
| + } |
| + if !a.MeterComplete(usage.MeterStoreEgressGB) || !a.Closed { |
| + t.Fatalf("a month whose first hour is evidenced must close complete: complete %v closed %v", a.MeterComplete(usage.MeterStoreEgressGB), a.Closed) |
| + } |
| + ev, err := a.Evidence.Load(ctx, fx.rel, "2026-09") |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := ev.Verify(ctx, fx.rel); err != nil { |
| + t.Fatalf("unchanged: %v", err) |
| + } |
| + if err := objstore.WithdrawBucketHourEvidence(ctx, fx.rel, foldBucketA, evidenceHour, "ops@keymail.dev", "the query covered the wrong region", time.Date(2026, 10, 6, 9, 0, 0, 0, time.UTC)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := ev.Verify(ctx, fx.rel); !errors.Is(err, usage.ErrEvidenceChanged) { |
| + t.Fatalf("a withdrawal after the close must read as changed evidence, got %v", err) |
| + } |
| +} |
| + |
| +// The doors: operators only; the record door accepts the first hour for |
| +// the ledger's owner and nothing else; each change leaves an audit entry. |
| +func TestBillingBucketEvidenceDoors(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + member := cliToken(t, fx, "paul@keymail.dev", "code-bucket-evidence-member") |
| + op := billingOperatorToken(t, fx, "ops@keymail.dev", "code-bucket-evidence-op") |
| + const bucket = "carlos-store-ops-shop" |
| + if err := objstore.OpenBucketOwner(ctx, fx.rel, bucket, fx.sqid, "shop", evidenceLooked, evidenceLooked, true); err != nil { |
| + t.Fatal(err) |
| + } |
| + record := map[string]any{"bucket": bucket, "hour": "2026-09-03T10:00:00Z", "account": fx.sqid, "app": "shop", "evidence": evidenceText} |
| + |
| + // Not an operator: the 404 an unknown path gets, and nothing written. |
| + if status := rawStatus(t, http.MethodGet, fx.ts.URL+"/api/billing/bucket-evidence?bucket="+bucket, nil, member); status != http.StatusNotFound { |
| + t.Errorf("list as a member = %d", status) |
| + } |
| + if status := rawStatus(t, http.MethodPost, fx.ts.URL+"/api/billing/bucket-evidence", record, member); status != http.StatusNotFound { |
| + t.Errorf("record as a member = %d", status) |
| + } |
| + if status := rawStatus(t, http.MethodPost, fx.ts.URL+"/api/billing/bucket-evidence/withdraw", map[string]any{"bucket": bucket, "hour": "2026-09-03T10:00:00Z", "note": "n"}, member); status != http.StatusNotFound { |
| + t.Errorf("withdraw as a member = %d", status) |
| + } |
| + if e, _, err := objstore.ReadBucketEvidence(ctx, fx.rel, bucket); err != nil || e != nil { |
| + t.Fatalf("a refused door wrote: %+v %v", e, err) |
| + } |
| + |
| + // The list names the hour evidence can be recorded for. |
| + status, out := getJSON(t, fx.ts.URL+"/api/billing/bucket-evidence?bucket="+bucket, op) |
| + if status != http.StatusOK || out["held_hour"] != "2026-09-03T10:00:00Z" || out["complete_since"] != "2026-09-03T10:20:00Z" { |
| + t.Fatalf("list = %d %v", status, out) |
| + } |
| + if status, _ := getJSON(t, fx.ts.URL+"/api/billing/bucket-evidence?bucket=carlos-store-nobody", op); status != http.StatusNotFound { |
| + t.Errorf("list for a bucket with no ledger = %d", status) |
| + } |
| + |
| + // Refused: another hour, another app, a malformed hour. |
| + for name, change := range map[string]map[string]any{ |
| + "the next hour": {"hour": "2026-09-03T11:00:00Z"}, |
| + "another app": {"app": "blog"}, |
| + } { |
| + body := map[string]any{} |
| + for k, v := range record { |
| + body[k] = v |
| + } |
| + for k, v := range change { |
| + body[k] = v |
| + } |
| + if status, _ := postJSON(t, fx.ts.URL+"/api/billing/bucket-evidence", body, op); status != http.StatusConflict { |
| + t.Errorf("%s = %d, want 409", name, status) |
| + } |
| + } |
| + bad := map[string]any{"bucket": bucket, "hour": "2026-09-03T10:20:00Z", "account": fx.sqid, "app": "shop", "evidence": evidenceText} |
| + if status, _ := postJSON(t, fx.ts.URL+"/api/billing/bucket-evidence", bad, op); status != http.StatusBadRequest { |
| + t.Errorf("an hour that is not on the hour = %d, want 400", status) |
| + } |
| + if e, _, err := objstore.ReadBucketEvidence(ctx, fx.rel, bucket); err != nil || e != nil { |
| + t.Fatalf("a refused record wrote: %+v %v", e, err) |
| + } |
| + |
| + if status, out := postJSON(t, fx.ts.URL+"/api/billing/bucket-evidence", record, op); status != http.StatusOK || out["recorded"] != true { |
| + t.Fatalf("record = %d %v", status, out) |
| + } |
| + e, _, err := objstore.ReadBucketEvidence(ctx, fx.rel, bucket) |
| + if err != nil || e == nil || len(e.Hours) != 1 || e.Hours[0].By != "ops@keymail.dev" || e.Hours[0].Evidence != evidenceText { |
| + t.Fatalf("the record: %+v %v", e, err) |
| + } |
| + other := map[string]any{"bucket": bucket, "hour": "2026-09-03T10:00:00Z", "account": fx.sqid, "app": "shop", "evidence": "a different story"} |
| + if status, _ := postJSON(t, fx.ts.URL+"/api/billing/bucket-evidence", other, op); status != http.StatusConflict { |
| + t.Errorf("a different finding over a standing entry = %d, want 409", status) |
| + } |
| + status, out = getJSON(t, fx.ts.URL+"/api/billing/bucket-evidence?bucket="+bucket, op) |
| + rows, _ := out["evidence"].([]any) |
| + if status != http.StatusOK || len(rows) != 1 || rows[0].(map[string]any)["by"] != "ops@keymail.dev" { |
| + t.Fatalf("list after record = %d %v", status, out) |
| + } |
| + |
| + withdraw := map[string]any{"bucket": bucket, "hour": "2026-09-03T10:00:00Z", "note": "the query covered the wrong region"} |
| + if status, _ := postJSON(t, fx.ts.URL+"/api/billing/bucket-evidence/withdraw", map[string]any{"bucket": bucket, "hour": "2026-09-03T10:00:00Z"}, op); status != http.StatusBadRequest { |
| + t.Errorf("withdraw with no note = %d, want 400", status) |
| + } |
| + if status, out := postJSON(t, fx.ts.URL+"/api/billing/bucket-evidence/withdraw", withdraw, op); status != http.StatusOK || out["withdrawn"] != true { |
| + t.Fatalf("withdraw = %d %v", status, out) |
| + } |
| + if status, _ := postJSON(t, fx.ts.URL+"/api/billing/bucket-evidence/withdraw", withdraw, op); status != http.StatusNotFound { |
| + t.Errorf("withdraw twice = %d, want 404", status) |
| + } |
| + e, _, _ = objstore.ReadBucketEvidence(ctx, fx.rel, bucket) |
| + if len(e.Hours) != 1 || e.Hours[0].WithdrawnAt == nil || e.Hours[0].WithdrawnBy != "ops@keymail.dev" { |
| + t.Fatalf("the withdrawal: %+v", e.Hours) |
| + } |
| + |
| + // One audit entry for the record and one for the withdrawal. |
| + var recorded, withdrawn int |
| + if err := fx.st.db.QueryRow(`SELECT COUNT(*) FROM audit WHERE action='billing-bucket-evidence'`).Scan(&recorded); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := fx.st.db.QueryRow(`SELECT COUNT(*) FROM audit WHERE action='billing-bucket-evidence-withdraw'`).Scan(&withdrawn); err != nil { |
| + t.Fatal(err) |
| + } |
| + if recorded != 1 || withdrawn != 1 { |
| + t.Fatalf("audit entries: %d recorded, %d withdrawn, want 1 and 1", recorded, withdrawn) |
| + } |
| +} |
diff --git a/internal/console/server.go b/internal/console/server.go| index cded07cf9..d98a5ee23 100644 |
| --- a/internal/console/server.go |
| +++ b/internal/console/server.go |
| @@ -1154,6 +1154,9 @@ func NewServer(cfg Config, st *Store) (*Server, error) { |
| s.mux.HandleFunc("POST /api/billing/reviews/resolve", s.handleAPIBillingReviewResolve) |
| s.mux.HandleFunc("GET /api/billing/house", s.handleAPIBillingHouseList) |
| s.mux.HandleFunc("POST /api/billing/house", s.handleAPIBillingHouseSet) |
| + s.mux.HandleFunc("GET /api/billing/bucket-evidence", s.handleAPIBillingBucketEvidenceList) |
| + s.mux.HandleFunc("POST /api/billing/bucket-evidence", s.handleAPIBillingBucketEvidenceRecord) |
| + s.mux.HandleFunc("POST /api/billing/bucket-evidence/withdraw", s.handleAPIBillingBucketEvidenceWithdraw) |
| s.mux.HandleFunc("POST /a/{sqid}/errors/level", s.handleAccountErrorLevel) |
| s.mux.HandleFunc("POST /a/{sqid}/analytics/setting", s.handleAccountAnalyticsSetting) |
| // Go 1.22's mux prefers this literal over apps/{app} below — exactly why |
diff --git a/internal/console/storeegressfold.go b/internal/console/storeegressfold.go| index 909f1f4a7..80fa97438 100644 |
| --- a/internal/console/storeegressfold.go |
| +++ b/internal/console/storeegressfold.go |
| @@ -32,7 +32,10 @@ import ( |
| // else; |
| // - no charged line was unclassifiable (awsbilling.IsHold); |
| // - every charged bucket-hour resolved to exactly one owner, or belongs |
| -// to a bucket this deployment declares its own. |
| +// to a bucket this deployment declares its own. The one hour a ledger |
| +// cannot answer for by itself, the hour it began vouching in, resolves |
| +// only on an operator's recorded evidence (objstore/hourevidence.go), |
| +// and holds without it. |
| // |
| // Anything less folds what could be attributed as a lower bound and leaves |
| // the source at zero days, so the freeze is refused. No byte reaches a |
| @@ -67,6 +70,10 @@ type storeEgressResult struct { |
| Unowned []string |
| // Holds counts charged lines the classifier could not count, by reason. |
| Holds map[string]int |
| + // Evidenced names the charged bucket-hours that resolved on an |
| + // operator's recorded evidence and not on the ledger alone, with who |
| + // recorded it. They are counted, and they are said. |
| + Evidenced []string |
| // PlatformBytes is charged transfer from the deployment's own buckets: |
| // the platform's cost, billed to nobody. |
| PlatformBytes int64 |
| @@ -165,6 +172,21 @@ func (s *Server) foldStoreEgress(ctx context.Context, period time.Time, meters m |
| ledgers[k.bucket] = ledger |
| } |
| acct, _, ok := ledger.OwnerAt(k.hour) |
| + if !ok { |
| + // The hour the ledger began vouching in holds unless a person |
| + // recorded what the provider's trail shows for it. |
| + eacct, _, by, eok, eerr := objstore.EvidencedOwnerAt(ctx, s.cfg.Releases, ledger, k.bucket, k.hour) |
| + switch { |
| + case eerr != nil: |
| + // As with the ledger: evidence that cannot be read is a |
| + // hold, not an absence. |
| + log.Printf("console: store egress %s: evidence for %s: %v", awsbilling.PeriodLabel(period), k.bucket, eerr) |
| + ownerUnread = true |
| + case eok: |
| + acct, ok = eacct, true |
| + res.Evidenced = append(res.Evidenced, k.bucket+" "+k.hour.Format(time.RFC3339)+" by "+by) |
| + } |
| + } |
| if !ok || !release.ValidAccount(acct) { |
| unowned[k.bucket] = true |
| continue |
| @@ -199,6 +221,9 @@ func (s *Server) foldStoreEgress(ctx context.Context, period time.Time, meters m |
| res.Reason = storeEgressNotFinal |
| default: |
| res.Complete, res.HoldClose = true, false |
| + if len(res.Evidenced) > 0 { |
| + log.Printf("console: store egress %s: complete; resolved on operator evidence: %v", awsbilling.PeriodLabel(period), res.Evidenced) |
| + } |
| return res |
| } |
| log.Printf("console: store egress %s: %s", awsbilling.PeriodLabel(period), res.describe()) |
diff --git a/internal/objstore/hourevidence.go b/internal/objstore/hourevidence.go| new file mode 100644 |
| index 000000000..e41458bc2 |
| --- /dev/null |
| +++ b/internal/objstore/hourevidence.go |
| @@ -0,0 +1,335 @@ |
| +package objstore |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "os" |
| + "strings" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// Operator evidence for a bucket's partial first hour. |
| +// |
| +// A complete ownership ledger answers for every hour that begins at or |
| +// after the instant it began vouching (BucketOwnership.OwnerAt). The hour |
| +// it began vouching IN does not resolve: the probe that found no bucket and |
| +// no principal says nothing about the minutes before it, and a charged line |
| +// is hourly. A charged line in that hour holds the month. |
| +// |
| +// Nothing in the platform can release that hold, because the platform did |
| +// not look earlier. A person can: the provider's own audit trail for the |
| +// bucket name shows whether anything of that name existed earlier in the |
| +// hour. This file is where that person's finding is recorded: for one |
| +// bucket and one hour, who is shown to have owned it, on what evidence, by |
| +// whom and when. The store-transfer fold honours a standing entry and |
| +// names it in what it reports. |
| +// |
| +// It is not a waiver and it cannot be used as one: |
| +// |
| +// - an entry is accepted, and honoured, only for the one hour the ledger |
| +// began vouching in. Every other unresolved hour (no ledger, a ledger |
| +// that claims no completeness, an hour two apps share) stays held |
| +// whatever is recorded here; |
| +// - it can only name the owner the ledger itself names for that hour. |
| +// The evidence says "nobody else held the name earlier in this hour"; |
| +// it never chooses who pays; |
| +// - the conditions are checked again each time an entry is read for a |
| +// fold, against the ledger as it then stands; |
| +// - entries are append-only. One can be withdrawn, which is recorded |
| +// beside it, and the hour then holds again. |
| + |
| +// BucketEvidencePrefix is where the evidence records live, in the |
| +// deployment bucket beside the ownership ledgers. |
| +const BucketEvidencePrefix = "usage/bucket-evidence/" |
| + |
| +// BucketEvidenceKey is the evidence record of one bucket. |
| +func BucketEvidenceKey(bucket string) string { return BucketEvidencePrefix + bucket + ".json" } |
| + |
| +// maxEvidenceText bounds what a person may write in one entry. |
| +const maxEvidenceText = 2000 |
| + |
| +// HourEvidence is one person's finding about one bucket-hour. |
| +type HourEvidence struct { |
| + // Hour is the first instant of the UTC hour. |
| + Hour time.Time `json:"hour"` |
| + Account string `json:"account_sqid"` |
| + App string `json:"app"` |
| + // Evidence is what was checked and what it showed, in the person's |
| + // words: which trail, which query, which event ids. |
| + Evidence string `json:"evidence"` |
| + By string `json:"by"` |
| + At time.Time `json:"at"` |
| + // A withdrawn entry stands for nothing. It is kept, with who withdrew |
| + // it and why. |
| + WithdrawnAt *time.Time `json:"withdrawn_at,omitempty"` |
| + WithdrawnBy string `json:"withdrawn_by,omitempty"` |
| + WithdrawnNote string `json:"withdrawn_note,omitempty"` |
| +} |
| + |
| +func (h HourEvidence) standing() bool { return h.WithdrawnAt == nil } |
| + |
| +// BucketEvidence is one bucket's evidence record. |
| +type BucketEvidence struct { |
| + V int `json:"v"` // 1 |
| + Bucket string `json:"bucket"` |
| + Hours []HourEvidence `json:"hours"` |
| +} |
| + |
| +var ( |
| + // ErrEvidenceInvalid: the entry is malformed. |
| + ErrEvidenceInvalid = errors.New("objstore: invalid bucket-hour evidence") |
| + // ErrEvidenceNotFirstHour: the hour is not the one the bucket's ledger |
| + // began vouching in, or the ledger does not name that owner, and only |
| + // that owner, for it. Evidence cannot release any other hold. |
| + ErrEvidenceNotFirstHour = errors.New("objstore: evidence is accepted only for the hour a complete ledger began vouching in, and only for the owner that ledger names") |
| + // ErrEvidenceExists: a standing entry for the hour says something else. |
| + ErrEvidenceExists = errors.New("objstore: a standing entry for that bucket-hour says something else; withdraw it first") |
| + // ErrEvidenceNotFound: there is no standing entry to withdraw. |
| + ErrEvidenceNotFound = errors.New("objstore: no standing evidence for that bucket-hour") |
| +) |
| + |
| +func hourAligned(t time.Time) bool { |
| + return !t.IsZero() && t.Equal(t.UTC().Truncate(time.Hour)) |
| +} |
| + |
| +func (e *BucketEvidence) validate(bucket string) error { |
| + if e.V != 1 || e.Bucket != bucket { |
| + return fmt.Errorf("objstore: %s: version %d or bucket %q is not what the key names", BucketEvidenceKey(bucket), e.V, e.Bucket) |
| + } |
| + standing := map[int64]bool{} |
| + for i, h := range e.Hours { |
| + text := strings.TrimSpace(h.Evidence) |
| + switch { |
| + case !hourAligned(h.Hour), !release.ValidAccount(h.Account), !release.ValidApp(h.App), |
| + text == "", len(h.Evidence) > maxEvidenceText, strings.TrimSpace(h.By) == "", validOwnershipTime(h.At) != nil: |
| + return fmt.Errorf("objstore: %s: entry %d is malformed", BucketEvidenceKey(bucket), i) |
| + case h.WithdrawnAt != nil && (validOwnershipTime(*h.WithdrawnAt) != nil || h.WithdrawnAt.Before(h.At) || strings.TrimSpace(h.WithdrawnBy) == "" || strings.TrimSpace(h.WithdrawnNote) == ""): |
| + return fmt.Errorf("objstore: %s: entry %d has a malformed withdrawal", BucketEvidenceKey(bucket), i) |
| + case h.WithdrawnAt == nil && (h.WithdrawnBy != "" || h.WithdrawnNote != ""): |
| + return fmt.Errorf("objstore: %s: entry %d has a malformed withdrawal", BucketEvidenceKey(bucket), i) |
| + } |
| + if h.standing() { |
| + if standing[h.Hour.Unix()] { |
| + return fmt.Errorf("objstore: %s: two standing entries for %s", BucketEvidenceKey(bucket), h.Hour.Format(time.RFC3339)) |
| + } |
| + standing[h.Hour.Unix()] = true |
| + } |
| + } |
| + return nil |
| +} |
| + |
| +// ReadBucketEvidence reads a bucket's evidence record. Absent is |
| +// (nil, "", nil). A record that will not parse or validate is an error: |
| +// the fold holds on it instead of reading it as "no evidence". |
| +func ReadBucketEvidence(ctx context.Context, s release.Store, bucket string) (*BucketEvidence, string, error) { |
| + if !ValidBucketName(bucket) { |
| + return nil, "", fmt.Errorf("objstore: invalid bucket name %q", bucket) |
| + } |
| + raw, etag, err := s.Get(ctx, BucketEvidenceKey(bucket)) |
| + if err != nil { |
| + if errors.Is(err, os.ErrNotExist) { |
| + return nil, "", nil |
| + } |
| + return nil, "", err |
| + } |
| + var e BucketEvidence |
| + if err := json.Unmarshal(raw, &e); err != nil { |
| + return nil, "", fmt.Errorf("objstore: %s: %w", BucketEvidenceKey(bucket), err) |
| + } |
| + if err := e.validate(bucket); err != nil { |
| + return nil, "", err |
| + } |
| + return &e, etag, nil |
| +} |
| + |
| +// firstHourOwner answers who the ledger names for the hour it began |
| +// vouching in, and only for that hour: ok is false for every other hour, |
| +// for a ledger that claims no completeness, and when the intervals that |
| +// touch the hour do not all name one app. |
| +func firstHourOwner(ledger *BucketOwnership, hour time.Time) (acct, app string, ok bool) { |
| + if ledger == nil || ledger.CompleteSince == nil || !hourAligned(hour) { |
| + return "", "", false |
| + } |
| + from, to := hour.UTC(), hour.UTC().Add(time.Hour) |
| + // The held hour is the one that began before the ledger looked and |
| + // ended after. A ledger that began vouching exactly on the hour |
| + // answers for that hour itself (OwnerAt) and has no held hour. |
| + if !ledger.CompleteSince.After(from) || !ledger.CompleteSince.Before(to) { |
| + return "", "", false |
| + } |
| + for _, iv := range ledger.Intervals { |
| + if !iv.overlaps(from, to) || (iv.Until != nil && iv.Until.Equal(iv.From)) { |
| + continue |
| + } |
| + if acct == "" { |
| + acct, app = iv.Account, iv.App |
| + continue |
| + } |
| + if iv.Account != acct || iv.App != app { |
| + return "", "", false |
| + } |
| + } |
| + return acct, app, acct != "" |
| +} |
| + |
| +// OwnerAt answers who a standing entry shows owned the bucket for hour, |
| +// given the bucket's ledger as it stands now. by is who recorded it. |
| +func (e *BucketEvidence) OwnerAt(ledger *BucketOwnership, hour time.Time) (acct, app, by string, ok bool) { |
| + if e == nil { |
| + return "", "", "", false |
| + } |
| + lacct, lapp, lok := firstHourOwner(ledger, hour) |
| + if !lok { |
| + return "", "", "", false |
| + } |
| + for _, h := range e.Hours { |
| + if h.standing() && h.Hour.Equal(hour) { |
| + if h.Account != lacct || h.App != lapp { |
| + return "", "", "", false |
| + } |
| + return h.Account, h.App, h.By, true |
| + } |
| + } |
| + return "", "", "", false |
| +} |
| + |
| +// evidenceAppendOnly checks that next only extends prev: every entry of |
| +// prev is still there, in place and unchanged, except that a standing |
| +// entry may have been withdrawn. |
| +func evidenceAppendOnly(prev, next *BucketEvidence) error { |
| + if prev == nil { |
| + return nil |
| + } |
| + if len(next.Hours) < len(prev.Hours) { |
| + return fmt.Errorf("objstore: evidence %s: a write may not remove an entry", prev.Bucket) |
| + } |
| + for i, p := range prev.Hours { |
| + n := next.Hours[i] |
| + if !n.Hour.Equal(p.Hour) || n.Account != p.Account || n.App != p.App || n.Evidence != p.Evidence || n.By != p.By || !n.At.Equal(p.At) { |
| + return fmt.Errorf("objstore: evidence %s: entry %d was rewritten", prev.Bucket, i) |
| + } |
| + if p.WithdrawnAt != nil && (n.WithdrawnAt == nil || !n.WithdrawnAt.Equal(*p.WithdrawnAt) || n.WithdrawnBy != p.WithdrawnBy || n.WithdrawnNote != p.WithdrawnNote) { |
| + return fmt.Errorf("objstore: evidence %s: the withdrawal of entry %d was changed", prev.Bucket, i) |
| + } |
| + } |
| + return nil |
| +} |
| + |
| +var errEvidenceUnchanged = errors.New("objstore: evidence unchanged") |
| + |
| +func writeBucketEvidence(ctx context.Context, s release.Store, bucket string, change func(*BucketEvidence) (*BucketEvidence, error)) error { |
| + for attempt := 0; attempt < 3; attempt++ { |
| + prev, etag, err := ReadBucketEvidence(ctx, s, bucket) |
| + if err != nil { |
| + return err |
| + } |
| + working := &BucketEvidence{V: 1, Bucket: bucket} |
| + if prev != nil { |
| + working.Hours = append([]HourEvidence(nil), prev.Hours...) |
| + } |
| + next, err := change(working) |
| + if errors.Is(err, errEvidenceUnchanged) { |
| + return nil |
| + } |
| + if err != nil { |
| + return err |
| + } |
| + if err := next.validate(bucket); err != nil { |
| + return fmt.Errorf("%w: %v", ErrEvidenceInvalid, err) |
| + } |
| + if err := evidenceAppendOnly(prev, next); err != nil { |
| + return err |
| + } |
| + body, err := json.MarshalIndent(next, "", " ") |
| + if err != nil { |
| + return err |
| + } |
| + if _, err := s.Put(ctx, BucketEvidenceKey(bucket), append(body, '\n'), etag); err == nil { |
| + return nil |
| + } else if !errors.Is(err, release.ErrCASConflict) { |
| + return err |
| + } |
| + } |
| + return fmt.Errorf("objstore: %s: lost CAS race three times", BucketEvidenceKey(bucket)) |
| +} |
| + |
| +// RecordBucketHourEvidence records that by has shown acct/app to be the |
| +// only holder of bucket in the UTC hour beginning at hour, and how. |
| +// |
| +// It is refused unless the bucket's ledger is complete, began vouching |
| +// inside that hour, and names acct/app and nobody else for it. Recording |
| +// the same finding again changes nothing; a different finding for an hour |
| +// that has a standing entry is refused until that entry is withdrawn. |
| +func RecordBucketHourEvidence(ctx context.Context, s release.Store, bucket string, hour time.Time, acct, app, evidence, by string, now time.Time) error { |
| + evidence, by, now = strings.TrimSpace(evidence), strings.TrimSpace(by), now.UTC() |
| + if !ValidBucketName(bucket) || !hourAligned(hour) || !release.ValidAccount(acct) || !release.ValidApp(app) || |
| + evidence == "" || len(evidence) > maxEvidenceText || by == "" || validOwnershipTime(now) != nil { |
| + return ErrEvidenceInvalid |
| + } |
| + hour = hour.UTC() |
| + ledger, _, err := ReadBucketOwnership(ctx, s, bucket) |
| + if err != nil { |
| + return err |
| + } |
| + if lacct, lapp, ok := firstHourOwner(ledger, hour); !ok || lacct != acct || lapp != app { |
| + return ErrEvidenceNotFirstHour |
| + } |
| + return writeBucketEvidence(ctx, s, bucket, func(e *BucketEvidence) (*BucketEvidence, error) { |
| + for _, h := range e.Hours { |
| + if !h.standing() || !h.Hour.Equal(hour) { |
| + continue |
| + } |
| + if h.Account == acct && h.App == app && h.Evidence == evidence { |
| + return nil, errEvidenceUnchanged |
| + } |
| + return nil, ErrEvidenceExists |
| + } |
| + e.Hours = append(e.Hours, HourEvidence{Hour: hour, Account: acct, App: app, Evidence: evidence, By: by, At: now}) |
| + return e, nil |
| + }) |
| +} |
| + |
| +// WithdrawBucketHourEvidence withdraws the standing entry for a |
| +// bucket-hour. The entry stays in the record with who withdrew it and why, |
| +// and the hour holds again. |
| +func WithdrawBucketHourEvidence(ctx context.Context, s release.Store, bucket string, hour time.Time, by, note string, now time.Time) error { |
| + by, note, now = strings.TrimSpace(by), strings.TrimSpace(note), now.UTC() |
| + if !ValidBucketName(bucket) || !hourAligned(hour) || by == "" || note == "" || len(note) > maxEvidenceText || validOwnershipTime(now) != nil { |
| + return ErrEvidenceInvalid |
| + } |
| + return writeBucketEvidence(ctx, s, bucket, func(e *BucketEvidence) (*BucketEvidence, error) { |
| + for i, h := range e.Hours { |
| + if h.standing() && h.Hour.Equal(hour) { |
| + at := now |
| + if at.Before(h.At) { |
| + at = h.At |
| + } |
| + e.Hours[i].WithdrawnAt, e.Hours[i].WithdrawnBy, e.Hours[i].WithdrawnNote = &at, by, note |
| + return e, nil |
| + } |
| + } |
| + return nil, ErrEvidenceNotFound |
| + }) |
| +} |
| + |
| +// EvidencedOwnerAt answers for an hour the ledger does not resolve by |
| +// itself: who a standing evidence entry shows owned the bucket, and who |
| +// recorded it. It reads the evidence record only when the hour is the one |
| +// the ledger began vouching in, so a month that does not rest on evidence |
| +// never reads, or pins, the record. |
| +func EvidencedOwnerAt(ctx context.Context, s release.Store, ledger *BucketOwnership, bucket string, hour time.Time) (acct, app, by string, ok bool, err error) { |
| + hour = hour.UTC().Truncate(time.Hour) |
| + if _, _, first := firstHourOwner(ledger, hour); !first { |
| + return "", "", "", false, nil |
| + } |
| + e, _, err := ReadBucketEvidence(ctx, s, bucket) |
| + if err != nil { |
| + return "", "", "", false, err |
| + } |
| + acct, app, by, ok = e.OwnerAt(ledger, hour) |
| + return acct, app, by, ok, nil |
| +} |
diff --git a/internal/objstore/hourevidence_test.go b/internal/objstore/hourevidence_test.go| new file mode 100644 |
| index 000000000..b46e82acc |
| --- /dev/null |
| +++ b/internal/objstore/hourevidence_test.go |
| @@ -0,0 +1,213 @@ |
| +package objstore |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +const evBucket = "carlos-store-tito-bab-titogo" |
| + |
| +var ( |
| + evLooked = time.Date(2026, 9, 3, 10, 20, 0, 0, time.UTC) // the ledger began vouching |
| + evFirstHour = time.Date(2026, 9, 3, 10, 0, 0, 0, time.UTC) |
| + evNow = time.Date(2026, 10, 2, 9, 0, 0, 0, time.UTC) |
| +) |
| + |
| +const evText = "CloudTrail eu-west-1, bucket name lookup 09:00 to 10:20 UTC: one CreateBucket at 10:20:04 (event 4f1c), nothing earlier" |
| + |
| +func evLedger(t *testing.T, s release.Store, minted bool) { |
| + t.Helper() |
| + if err := OpenBucketOwner(context.Background(), s, evBucket, "bab", "titogo", evLooked, evLooked, minted); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +// The door releases one hold and no other: the hour a complete ledger began |
| +// vouching in, for the owner that ledger names. Everything else it is |
| +// offered is refused and nothing is written. |
| +func TestBucketHourEvidenceIsAcceptedOnlyForTheLedgersFirstHour(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + rec := func(hour time.Time, acct, app string) error { |
| + return RecordBucketHourEvidence(ctx, s, evBucket, hour, acct, app, evText, "op@example.com", evNow) |
| + } |
| + // No ledger at all. |
| + if err := rec(evFirstHour, "bab", "titogo"); !errors.Is(err, ErrEvidenceNotFirstHour) { |
| + t.Fatalf("no ledger: %v", err) |
| + } |
| + evLedger(t, s, true) |
| + for name, c := range map[string]struct { |
| + hour time.Time |
| + acct, app string |
| + }{ |
| + "the hour after, which the ledger answers for itself": {evFirstHour.Add(time.Hour), "bab", "titogo"}, |
| + "the hour before, when nothing was looked at": {evFirstHour.Add(-time.Hour), "bab", "titogo"}, |
| + "another account": {evFirstHour, "bac", "titogo"}, |
| + "another app": {evFirstHour, "bab", "other"}, |
| + } { |
| + if err := rec(c.hour, c.acct, c.app); !errors.Is(err, ErrEvidenceNotFirstHour) { |
| + t.Errorf("%s: %v, want ErrEvidenceNotFirstHour", name, err) |
| + } |
| + } |
| + if err := rec(evLooked, "bab", "titogo"); !errors.Is(err, ErrEvidenceInvalid) { |
| + t.Errorf("an instant that is not the start of an hour: %v", err) |
| + } |
| + if err := RecordBucketHourEvidence(ctx, s, evBucket, evFirstHour, "bab", "titogo", " ", "op@example.com", evNow); !errors.Is(err, ErrEvidenceInvalid) { |
| + t.Errorf("no evidence text: %v", err) |
| + } |
| + if err := RecordBucketHourEvidence(ctx, s, evBucket, evFirstHour, "bab", "titogo", evText, "", evNow); !errors.Is(err, ErrEvidenceInvalid) { |
| + t.Errorf("nobody named: %v", err) |
| + } |
| + if e, _, err := ReadBucketEvidence(ctx, s, evBucket); err != nil || e != nil { |
| + t.Fatalf("a refused entry was written: %+v %v", e, err) |
| + } |
| + if err := rec(evFirstHour, "bab", "titogo"); err != nil { |
| + t.Fatalf("the first hour, the ledger's owner: %v", err) |
| + } |
| + ledger, _, _ := ReadBucketOwnership(ctx, s, evBucket) |
| + e, _, err := ReadBucketEvidence(ctx, s, evBucket) |
| + if err != nil || e == nil || len(e.Hours) != 1 { |
| + t.Fatalf("the record: %+v %v", e, err) |
| + } |
| + if acct, app, by, ok := e.OwnerAt(ledger, evFirstHour); !ok || acct != "bab" || app != "titogo" || by != "op@example.com" { |
| + t.Fatalf("OwnerAt = %s %s %s %v", acct, app, by, ok) |
| + } |
| + // The ledger still does not answer for that hour by itself. |
| + if _, _, ok := ledger.OwnerAt(evFirstHour); ok { |
| + t.Fatal("the ledger resolved its own first hour") |
| + } |
| + // And the evidence answers for no other hour. |
| + if _, _, _, ok := e.OwnerAt(ledger, evFirstHour.Add(time.Hour)); ok { |
| + t.Fatal("evidence answered for an hour it was not recorded for") |
| + } |
| +} |
| + |
| +// A ledger that claims no completeness (an adopted bucket, a store from |
| +// before ledgers) has no first hour to release: someone may have held |
| +// access it never saw, in any hour. |
| +func TestBucketHourEvidenceIsRefusedForALedgerThatClaimsNoCompleteness(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + evLedger(t, s, false) |
| + if err := RecordBucketHourEvidence(ctx, s, evBucket, evFirstHour, "bab", "titogo", evText, "op@example.com", evNow); !errors.Is(err, ErrEvidenceNotFirstHour) { |
| + t.Fatalf("an unproven ledger: %v", err) |
| + } |
| +} |
| + |
| +// A ledger that began vouching exactly on the hour answers for that hour |
| +// itself; there is nothing held and nothing to record. |
| +func TestBucketHourEvidenceIsRefusedWhenTheLedgerBeganOnTheHour(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + if err := OpenBucketOwner(ctx, s, evBucket, "bab", "titogo", evFirstHour, evFirstHour, true); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := RecordBucketHourEvidence(ctx, s, evBucket, evFirstHour, "bab", "titogo", evText, "op@example.com", evNow); !errors.Is(err, ErrEvidenceNotFirstHour) { |
| + t.Fatalf("a ledger that began on the hour: %v", err) |
| + } |
| +} |
| + |
| +// The conditions are checked again when the entry is read. If another app |
| +// is later shown to have touched the hour (a second interval on the |
| +// ledger), the entry stops answering: it said nobody else held the name, |
| +// and the ledger now says somebody did. |
| +func TestBucketHourEvidenceStopsAnsweringWhenTheLedgerNamesASecondHolder(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + evLedger(t, s, true) |
| + if err := RecordBucketHourEvidence(ctx, s, evBucket, evFirstHour, "bab", "titogo", evText, "op@example.com", evNow); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := OpenBucketOwner(ctx, s, evBucket, "bac", "other", evLooked.Add(10*time.Minute), evLooked.Add(10*time.Minute), false); err != nil { |
| + t.Fatal(err) |
| + } |
| + ledger, _, _ := ReadBucketOwnership(ctx, s, evBucket) |
| + e, _, _ := ReadBucketEvidence(ctx, s, evBucket) |
| + if _, _, _, ok := e.OwnerAt(ledger, evFirstHour); ok { |
| + t.Fatal("evidence still answered for an hour the ledger now shows two apps in") |
| + } |
| +} |
| + |
| +// Recording the same finding twice changes nothing. A different finding |
| +// over a standing entry is refused. A withdrawal is recorded beside the |
| +// entry, the hour holds again, and a new finding can then be recorded. |
| +func TestBucketHourEvidenceIsAppendOnly(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + evLedger(t, s, true) |
| + rec := func(text, by string, at time.Time) error { |
| + return RecordBucketHourEvidence(ctx, s, evBucket, evFirstHour, "bab", "titogo", text, by, at) |
| + } |
| + if err := rec(evText, "op@example.com", evNow); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := rec(evText, "second@example.com", evNow.Add(time.Hour)); err != nil { |
| + t.Fatalf("the same finding again: %v", err) |
| + } |
| + e, _, _ := ReadBucketEvidence(ctx, s, evBucket) |
| + if len(e.Hours) != 1 || e.Hours[0].By != "op@example.com" || !e.Hours[0].At.Equal(evNow) { |
| + t.Fatalf("a repeated finding rewrote the entry: %+v", e.Hours) |
| + } |
| + if err := rec("a different story", "second@example.com", evNow.Add(time.Hour)); !errors.Is(err, ErrEvidenceExists) { |
| + t.Fatalf("a different finding over a standing entry: %v", err) |
| + } |
| + if err := WithdrawBucketHourEvidence(ctx, s, evBucket, evFirstHour.Add(time.Hour), "op@example.com", "wrong hour", evNow); !errors.Is(err, ErrEvidenceNotFound) { |
| + t.Fatalf("withdrawing an hour with no entry: %v", err) |
| + } |
| + if err := WithdrawBucketHourEvidence(ctx, s, evBucket, evFirstHour, "op@example.com", "", evNow); !errors.Is(err, ErrEvidenceInvalid) { |
| + t.Fatalf("withdrawing with no reason: %v", err) |
| + } |
| + if err := WithdrawBucketHourEvidence(ctx, s, evBucket, evFirstHour, "second@example.com", "the trail query covered the wrong region", evNow.Add(2*time.Hour)); err != nil { |
| + t.Fatal(err) |
| + } |
| + ledger, _, _ := ReadBucketOwnership(ctx, s, evBucket) |
| + e, _, _ = ReadBucketEvidence(ctx, s, evBucket) |
| + if len(e.Hours) != 1 || e.Hours[0].WithdrawnAt == nil || e.Hours[0].WithdrawnBy != "second@example.com" || e.Hours[0].Evidence != evText { |
| + t.Fatalf("the withdrawal: %+v", e.Hours) |
| + } |
| + if _, _, _, ok := e.OwnerAt(ledger, evFirstHour); ok { |
| + t.Fatal("a withdrawn entry still answered") |
| + } |
| + if err := WithdrawBucketHourEvidence(ctx, s, evBucket, evFirstHour, "second@example.com", "again", evNow.Add(3*time.Hour)); !errors.Is(err, ErrEvidenceNotFound) { |
| + t.Fatalf("withdrawing twice: %v", err) |
| + } |
| + if err := rec("eu-west-1 and us-east-1 trails, same window, one CreateBucket", "second@example.com", evNow.Add(4*time.Hour)); err != nil { |
| + t.Fatalf("a new finding after a withdrawal: %v", err) |
| + } |
| + e, _, _ = ReadBucketEvidence(ctx, s, evBucket) |
| + if len(e.Hours) != 2 || e.Hours[0].WithdrawnAt == nil || e.Hours[1].WithdrawnAt != nil { |
| + t.Fatalf("the record after a second finding: %+v", e.Hours) |
| + } |
| + if _, _, by, ok := e.OwnerAt(ledger, evFirstHour); !ok || by != "second@example.com" { |
| + t.Fatalf("the second finding does not answer: %s %v", by, ok) |
| + } |
| +} |
| + |
| +// A record edited by hand into something the door would not write is an |
| +// error to its reader, not "no evidence" and not evidence. |
| +func TestBucketEvidenceThatDoesNotValidateIsAnError(t *testing.T) { |
| + ctx := context.Background() |
| + for name, body := range map[string]string{ |
| + "two standing entries for one hour": `{"v":1,"bucket":"` + evBucket + `","hours":[ |
| + {"hour":"2026-09-03T10:00:00Z","account_sqid":"bab","app":"titogo","evidence":"a","by":"op","at":"2026-10-02T09:00:00Z"}, |
| + {"hour":"2026-09-03T10:00:00Z","account_sqid":"bac","app":"other","evidence":"b","by":"op","at":"2026-10-02T09:00:00Z"}]}`, |
| + "an hour that is not on the hour": `{"v":1,"bucket":"` + evBucket + `","hours":[ |
| + {"hour":"2026-09-03T10:20:00Z","account_sqid":"bab","app":"titogo","evidence":"a","by":"op","at":"2026-10-02T09:00:00Z"}]}`, |
| + "another bucket's record": `{"v":1,"bucket":"carlos-store-other","hours":[]}`, |
| + "no evidence text": `{"v":1,"bucket":"` + evBucket + `","hours":[ |
| + {"hour":"2026-09-03T10:00:00Z","account_sqid":"bab","app":"titogo","evidence":"","by":"op","at":"2026-10-02T09:00:00Z"}]}`, |
| + "not json": `{`, |
| + } { |
| + s := release.NewDirStore(t.TempDir()) |
| + if _, err := s.Put(ctx, BucketEvidenceKey(evBucket), []byte(body), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + if e, _, err := ReadBucketEvidence(ctx, s, evBucket); err == nil { |
| + t.Errorf("%s: read as %+v", name, e) |
| + } |
| + } |
| +} |