amadan
objstore: record bucket ownership history for the store-transfer meter
The AWS Cost and Usage Report names the bucket a charged transfer came from and the hour it happened in. Attributing that to an account needs who owned the bucket in that hour, which a store record (deleted at teardown) and the current bucket listing cannot answer. usage/buckets/ <bucket>.json is an append-only, effective-dated ledger, written with compare-and-swap: Grant opens an interval from the grant block's own time before any credential is delivered, Teardown closes it after the bucket is deleted, and no write may remove, move or reopen an interval. OwnerAt answers for an hour only when every interval touching it names one account and app; two owners or none is a hold for the caller. Stores provisioned before this change have no ledger and therefore hold the month until a re-grant backfills them from their grant time. Verified: go test ./internal/objstore passes; go build and go vet clean. Implemented by an Opus worker interrupted mid-task; adopted, reviewed and committed by the lead.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
3 files changed,
+720
−0
internal/objstore/ownership.go+338 −0internal/objstore/ownership_test.go+361 −0internal/objstore/walk.go+21 −0
diff --git a/internal/objstore/ownership.go b/internal/objstore/ownership.go| new file mode 100644 |
| index 000000000..4e8666f7b |
| --- /dev/null |
| +++ b/internal/objstore/ownership.go |
| @@ -0,0 +1,338 @@ |
| +package objstore |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "fmt" |
| + "os" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// Bucket ownership history (store-transfer meter; pricing spec P5 and the |
| +// 2026-09-30 ruling that store transfer bills only what AWS charged). |
| +// |
| +// The AWS Cost and Usage Report names the BUCKET a charged transfer came |
| +// from and the HOUR it happened in. Turning that into an invoice line needs |
| +// "which account owned this bucket in that hour", and nothing else in the |
| +// platform can answer it: a store record is deleted when its store is torn |
| +// down, and the current bucket listing says who owns a name today, not who |
| +// owned it last Tuesday. So ownership is recorded as it happens, in one |
| +// append-only, effective-dated object per bucket: |
| +// |
| +// usage/buckets/<bucket>.json |
| +// {"v":1,"bucket":"...","intervals":[{"account_sqid":"...","app":"...","from":"...","until":"..."}]} |
| +// |
| +// It lives in the deployment bucket under usage/, beside the other metering |
| +// records, and carries no customer data: a bucket name, an account sqid and |
| +// an app name. |
| +// |
| +// The rules that make it billing-grade: |
| +// |
| +// - Append-only. A write may add an interval, or set the Until of an open |
| +// one exactly once. It never removes an interval, moves a From, or |
| +// shortens or reopens a closed one. writeBucketOwnership checks every |
| +// write against the record it replaces and refuses anything else. |
| +// - Compare-and-swap, three attempts, against the ETag of the read the |
| +// change was computed from (the internal/usage/lifecycle.go discipline). |
| +// - Opened BEFORE credentials are delivered (Grant), from the grant |
| +// block's own time, so no tenant can move a byte out of a bucket before |
| +// the ledger names them. Closed AFTER the bucket is deleted (Teardown), |
| +// so no byte can leave it after the ledger stops naming them. |
| +// - OwnerAt answers only when every interval touching the hour names the |
| +// same account and app. Two owners in one hour, or none, is not an |
| +// answer, and the caller must hold rather than choose. |
| + |
| +// BucketOwnershipPrefix is where ownership ledgers live. |
| +const BucketOwnershipPrefix = "usage/buckets/" |
| + |
| +// BucketOwnershipKey is the ledger key for one bucket. |
| +func BucketOwnershipKey(bucket string) string { return BucketOwnershipPrefix + bucket + ".json" } |
| + |
| +// OwnerInterval is one span in which an account's app held a bucket. |
| +// Until is nil while the ownership is current. |
| +type OwnerInterval struct { |
| + Account string `json:"account_sqid"` |
| + App string `json:"app"` |
| + From time.Time `json:"from"` |
| + Until *time.Time `json:"until,omitempty"` |
| +} |
| + |
| +func (iv OwnerInterval) open() bool { return iv.Until == nil } |
| + |
| +// overlaps reports whether the interval touches [from, to). |
| +func (iv OwnerInterval) overlaps(from, to time.Time) bool { |
| + if !iv.From.Before(to) { |
| + return false |
| + } |
| + return iv.Until == nil || iv.Until.After(from) |
| +} |
| + |
| +// BucketOwnership is one bucket's ownership ledger. |
| +type BucketOwnership struct { |
| + V int `json:"v"` // 1 |
| + Bucket string `json:"bucket"` |
| + Intervals []OwnerInterval `json:"intervals"` |
| +} |
| + |
| +func validOwnershipTime(t time.Time) error { |
| + if t.IsZero() { |
| + return fmt.Errorf("zero timestamp") |
| + } |
| + if t.Location() != time.UTC { |
| + return fmt.Errorf("timestamp %s is not UTC", t) |
| + } |
| + return nil |
| +} |
| + |
| +// validate is the gate every ledger passes on read and on write. A ledger |
| +// that fails it is reported, never silently skipped: an unreadable ledger |
| +// read as "no owner" would still hold (safe), but read as "empty" by a |
| +// writer it would be overwritten, which would erase history. |
| +func (b *BucketOwnership) validate(bucket string) error { |
| + if b.V != 1 { |
| + return fmt.Errorf("objstore: ownership ledger %s: schema v%d is not v1", bucket, b.V) |
| + } |
| + if b.Bucket != bucket { |
| + return fmt.Errorf("objstore: ownership ledger %s names bucket %q", bucket, b.Bucket) |
| + } |
| + if len(b.Intervals) == 0 { |
| + return fmt.Errorf("objstore: ownership ledger %s has no intervals", bucket) |
| + } |
| + for i, iv := range b.Intervals { |
| + if !release.ValidAccount(iv.Account) || !release.ValidApp(iv.App) { |
| + return fmt.Errorf("objstore: ownership ledger %s: interval %d names %q/%q", bucket, i, iv.Account, iv.App) |
| + } |
| + if err := validOwnershipTime(iv.From); err != nil { |
| + return fmt.Errorf("objstore: ownership ledger %s: interval %d from: %v", bucket, i, err) |
| + } |
| + if iv.Until != nil { |
| + if err := validOwnershipTime(*iv.Until); err != nil { |
| + return fmt.Errorf("objstore: ownership ledger %s: interval %d until: %v", bucket, i, err) |
| + } |
| + if iv.Until.Before(iv.From) { |
| + return fmt.Errorf("objstore: ownership ledger %s: interval %d ends before it starts", bucket, i) |
| + } |
| + } |
| + } |
| + return nil |
| +} |
| + |
| +// ReadBucketOwnership reads a bucket's ledger. Absent is (nil, "", nil): |
| +// a bucket nobody has recorded owns nothing, which OwnerAt reports as "no |
| +// owner" and a billing caller must hold on. |
| +func ReadBucketOwnership(ctx context.Context, s release.Store, bucket string) (*BucketOwnership, string, error) { |
| + if !ValidBucketName(bucket) { |
| + return nil, "", fmt.Errorf("objstore: invalid bucket name %q", bucket) |
| + } |
| + raw, etag, err := s.Get(ctx, BucketOwnershipKey(bucket)) |
| + if err != nil { |
| + if errors.Is(err, os.ErrNotExist) { |
| + return nil, "", nil |
| + } |
| + return nil, "", err |
| + } |
| + var b BucketOwnership |
| + if err := json.Unmarshal(raw, &b); err != nil { |
| + return nil, "", fmt.Errorf("objstore: %s: %w", BucketOwnershipKey(bucket), err) |
| + } |
| + if err := b.validate(bucket); err != nil { |
| + return nil, "", err |
| + } |
| + return &b, etag, nil |
| +} |
| + |
| +// appendOnly checks that next only extends prev: every interval of prev is |
| +// still there, in place, with the same owner and From, and an Until that is |
| +// either unchanged or newly set on an interval that was open. |
| +func appendOnly(prev, next *BucketOwnership) error { |
| + if prev == nil { |
| + return nil |
| + } |
| + if len(next.Intervals) < len(prev.Intervals) { |
| + return fmt.Errorf("objstore: ownership ledger %s: a write may not remove an interval", prev.Bucket) |
| + } |
| + for i, p := range prev.Intervals { |
| + n := next.Intervals[i] |
| + if n.Account != p.Account || n.App != p.App || !n.From.Equal(p.From) { |
| + return fmt.Errorf("objstore: ownership ledger %s: interval %d was rewritten", prev.Bucket, i) |
| + } |
| + if p.Until != nil && (n.Until == nil || !n.Until.Equal(*p.Until)) { |
| + return fmt.Errorf("objstore: ownership ledger %s: closed interval %d was changed", prev.Bucket, i) |
| + } |
| + } |
| + return nil |
| +} |
| + |
| +// errOwnershipUnchanged tells writeBucketOwnership the change is already |
| +// in place, so there is nothing to write. |
| +var errOwnershipUnchanged = errors.New("objstore: ownership unchanged") |
| + |
| +// writeBucketOwnership applies change to the current ledger and writes the |
| +// result with compare-and-swap, three attempts. change receives nil for an |
| +// absent ledger and returns the ledger to write, or errOwnershipUnchanged. |
| +func writeBucketOwnership(ctx context.Context, s release.Store, bucket string, change func(*BucketOwnership) (*BucketOwnership, error)) error { |
| + for attempt := 0; attempt < 3; attempt++ { |
| + prev, etag, err := ReadBucketOwnership(ctx, s, bucket) |
| + if err != nil { |
| + return err |
| + } |
| + var working *BucketOwnership |
| + if prev != nil { |
| + cp := *prev |
| + cp.Intervals = append([]OwnerInterval(nil), prev.Intervals...) |
| + working = &cp |
| + } |
| + next, err := change(working) |
| + if errors.Is(err, errOwnershipUnchanged) { |
| + return nil |
| + } |
| + if err != nil { |
| + return err |
| + } |
| + if err := next.validate(bucket); err != nil { |
| + return err |
| + } |
| + if err := appendOnly(prev, next); err != nil { |
| + return err |
| + } |
| + body, err := json.MarshalIndent(next, "", " ") |
| + if err != nil { |
| + return err |
| + } |
| + if _, err := s.Put(ctx, BucketOwnershipKey(bucket), append(body, '\n'), etag); err == nil { |
| + return nil |
| + } else if !errors.Is(err, release.ErrCASConflict) { |
| + return err |
| + } |
| + } |
| + return fmt.Errorf("objstore: %s: lost CAS race three times", BucketOwnershipKey(bucket)) |
| +} |
| + |
| +// OpenBucketOwner records that acct/app owns bucket from `from` on. |
| +// |
| +// Idempotent: if an open interval already names acct/app it is left |
| +// exactly as it is (its From is never moved, earlier or later). Otherwise |
| +// a new open interval is appended. |
| +// |
| +// An open interval naming ANOTHER owner does not stop the append. Two apps |
| +// can legitimately be handed one adopted bucket by an operator, and |
| +// refusing here would break provisioning for them; instead both intervals |
| +// stand, OwnerAt reports every hour they share as unresolved, and the |
| +// store-transfer meter holds the month with the bucket named, which is |
| +// the outcome a person needs to see. |
| +func OpenBucketOwner(ctx context.Context, s release.Store, bucket, acct, app string, from time.Time) error { |
| + if !release.ValidAccount(acct) || !release.ValidApp(app) { |
| + return fmt.Errorf("objstore: open bucket owner: invalid owner %q/%q", acct, app) |
| + } |
| + from = from.UTC() |
| + if err := validOwnershipTime(from); err != nil { |
| + return fmt.Errorf("objstore: open bucket owner %s: %v", bucket, err) |
| + } |
| + return writeBucketOwnership(ctx, s, bucket, func(b *BucketOwnership) (*BucketOwnership, error) { |
| + if b == nil { |
| + b = &BucketOwnership{V: 1, Bucket: bucket} |
| + } |
| + for _, iv := range b.Intervals { |
| + if iv.open() && iv.Account == acct && iv.App == app { |
| + return nil, errOwnershipUnchanged |
| + } |
| + } |
| + b.Intervals = append(b.Intervals, OwnerInterval{Account: acct, App: app, From: from}) |
| + return b, nil |
| + }) |
| +} |
| + |
| +// CloseBucketOwner ends acct/app's open ownership of bucket at until. A |
| +// bucket with no ledger, or no open interval for acct/app, is already in |
| +// the state asked for, and nothing is written. An until earlier than the |
| +// interval's From (clock skew between consoles) closes it at its From: the |
| +// interval then covers nothing, which is the truthful reading of "ended no |
| +// later than it began". |
| +func CloseBucketOwner(ctx context.Context, s release.Store, bucket, acct, app string, until time.Time) error { |
| + until = until.UTC() |
| + if err := validOwnershipTime(until); err != nil { |
| + return fmt.Errorf("objstore: close bucket owner %s: %v", bucket, err) |
| + } |
| + return writeBucketOwnership(ctx, s, bucket, func(b *BucketOwnership) (*BucketOwnership, error) { |
| + if b == nil { |
| + return nil, errOwnershipUnchanged |
| + } |
| + changed := false |
| + for i, iv := range b.Intervals { |
| + if iv.open() && iv.Account == acct && iv.App == app { |
| + end := until |
| + if end.Before(iv.From) { |
| + end = iv.From |
| + } |
| + b.Intervals[i].Until = &end |
| + changed = true |
| + } |
| + } |
| + if !changed { |
| + return nil, errOwnershipUnchanged |
| + } |
| + return b, nil |
| + }) |
| +} |
| + |
| +// OwnerAt answers who owned the bucket for the whole UTC hour containing |
| +// hour. ok is true only when at least one interval touches the hour and |
| +// every interval that does names the same account and app. |
| +// |
| +// An hour only partly covered by its one owner (the store was provisioned |
| +// at 10:20, or deleted at 10:40) still resolves to that owner: the ledger |
| +// is opened before credentials exist and closed after the bucket is gone, |
| +// so no one else could have moved bytes out of it in the rest of that hour. |
| +func (b *BucketOwnership) OwnerAt(hour time.Time) (acct, app string, ok bool) { |
| + if b == nil { |
| + return "", "", false |
| + } |
| + from := hour.UTC().Truncate(time.Hour) |
| + to := from.Add(time.Hour) |
| + for _, iv := range b.Intervals { |
| + if !iv.overlaps(from, to) { |
| + continue |
| + } |
| + if iv.Until != nil && iv.Until.Equal(iv.From) { |
| + continue // an interval that covers nothing touches nothing |
| + } |
| + if acct == "" { |
| + acct, app = iv.Account, iv.App |
| + continue |
| + } |
| + if iv.Account != acct || iv.App != app { |
| + return "", "", false |
| + } |
| + } |
| + return acct, app, acct != "" |
| +} |
| + |
| +// BucketOwners is a set of loaded ledgers keyed by bucket name, for callers |
| +// that ask about many hours of many buckets. |
| +type BucketOwners map[string]*BucketOwnership |
| + |
| +// OwnerAt answers for one bucket and hour; a bucket with no loaded ledger |
| +// has no owner. |
| +func (o BucketOwners) OwnerAt(bucket string, hour time.Time) (acct, app string, ok bool) { |
| + return o[bucket].OwnerAt(hour) |
| +} |
| + |
| +// ownershipNow is Teardown's clock for closing an interval. Teardown takes |
| +// no time parameter and has many callers; a package clock keeps them all |
| +// unchanged and lets tests pin it. |
| +var ownershipNow = time.Now |
| + |
| +// grantOwnershipFrom is the instant a grant's ownership interval opens: |
| +// the grant block's own time, which is written once, when the bucket is |
| +// chosen and before anything is created or delivered. A record from before |
| +// the block carried a time falls back to now, which can only open the |
| +// interval later, never earlier. |
| +func grantOwnershipFrom(g *GrantBlock, now time.Time) time.Time { |
| + if g == nil || g.At.IsZero() { |
| + return now.UTC() |
| + } |
| + return g.At.UTC() |
| +} |
diff --git a/internal/objstore/ownership_test.go b/internal/objstore/ownership_test.go| new file mode 100644 |
| index 000000000..0c6ff5bce |
| --- /dev/null |
| +++ b/internal/objstore/ownership_test.go |
| @@ -0,0 +1,361 @@ |
| +package objstore |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "errors" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +const ledgerBucket = "carlos-store-tito-bab-titogo" |
| + |
| +var t0 = time.Date(2026, 9, 3, 10, 20, 0, 0, time.UTC) |
| + |
| +func mustLedger(t *testing.T, s release.Store, bucket string) *BucketOwnership { |
| + t.Helper() |
| + b, _, err := ReadBucketOwnership(context.Background(), s, bucket) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if b == nil { |
| + t.Fatalf("no ledger for %s", bucket) |
| + } |
| + return b |
| +} |
| + |
| +func TestOpenBucketOwnerIsIdempotentAndNeverMovesFrom(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0); err != nil { |
| + t.Fatal(err) |
| + } |
| + // Earlier and later opens for the same owner change nothing. |
| + for _, at := range []time.Time{t0.Add(-48 * time.Hour), t0.Add(48 * time.Hour)} { |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", at); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + b := mustLedger(t, s, ledgerBucket) |
| + if len(b.Intervals) != 1 || !b.Intervals[0].From.Equal(t0) || b.Intervals[0].Until != nil { |
| + t.Fatalf("intervals = %+v, want one open interval from %s", b.Intervals, t0) |
| + } |
| +} |
| + |
| +func TestCloseBucketOwner(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + // No ledger: nothing to close, nothing written. |
| + if err := CloseBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0); err != nil { |
| + t.Fatal(err) |
| + } |
| + if b, _, _ := ReadBucketOwnership(ctx, s, ledgerBucket); b != nil { |
| + t.Fatal("closing an absent ledger must not create one") |
| + } |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0); err != nil { |
| + t.Fatal(err) |
| + } |
| + end := t0.Add(5 * time.Hour) |
| + if err := CloseBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", end); err != nil { |
| + t.Fatal(err) |
| + } |
| + // A second close, later, must not move the end: a closed interval is |
| + // never changed. |
| + if err := CloseBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", end.Add(time.Hour)); err != nil { |
| + t.Fatal(err) |
| + } |
| + b := mustLedger(t, s, ledgerBucket) |
| + if len(b.Intervals) != 1 || b.Intervals[0].Until == nil || !b.Intervals[0].Until.Equal(end) { |
| + t.Fatalf("intervals = %+v, want closed at %s", b.Intervals, end) |
| + } |
| + // Reopening appends; the closed interval stands. |
| + again := t0.Add(72 * time.Hour) |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", again); err != nil { |
| + t.Fatal(err) |
| + } |
| + b = mustLedger(t, s, ledgerBucket) |
| + if len(b.Intervals) != 2 || !b.Intervals[0].Until.Equal(end) || !b.Intervals[1].From.Equal(again) || b.Intervals[1].Until != nil { |
| + t.Fatalf("intervals = %+v", b.Intervals) |
| + } |
| +} |
| + |
| +func TestCloseBeforeFromClampsToEmpty(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := CloseBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0.Add(-time.Hour)); err != nil { |
| + t.Fatal(err) |
| + } |
| + b := mustLedger(t, s, ledgerBucket) |
| + if !b.Intervals[0].Until.Equal(t0) { |
| + t.Fatalf("until = %s, want clamped to from", b.Intervals[0].Until) |
| + } |
| + if _, _, ok := b.OwnerAt(t0); ok { |
| + t.Fatal("an empty interval must not own any hour") |
| + } |
| +} |
| + |
| +func TestOwnerAt(t *testing.T) { |
| + end := time.Date(2026, 9, 3, 14, 40, 0, 0, time.UTC) |
| + b := &BucketOwnership{V: 1, Bucket: ledgerBucket, Intervals: []OwnerInterval{ |
| + {Account: "bab", App: "titogo", From: t0, Until: &end}, |
| + }} |
| + cases := []struct { |
| + hour time.Time |
| + ok bool |
| + }{ |
| + {time.Date(2026, 9, 3, 9, 0, 0, 0, time.UTC), false}, // before the store existed |
| + {time.Date(2026, 9, 3, 10, 0, 0, 0, time.UTC), true}, // provisioned at 10:20 |
| + {time.Date(2026, 9, 3, 12, 0, 0, 0, time.UTC), true}, // whole hour |
| + {time.Date(2026, 9, 3, 14, 0, 0, 0, time.UTC), true}, // deleted at 14:40 |
| + {time.Date(2026, 9, 3, 15, 0, 0, 0, time.UTC), false}, // after deletion |
| + } |
| + for _, c := range cases { |
| + acct, app, ok := b.OwnerAt(c.hour) |
| + if ok != c.ok || (ok && (acct != "bab" || app != "titogo")) { |
| + t.Errorf("OwnerAt(%s) = %q %q %v, want ok=%v", c.hour.Format("15:04"), acct, app, ok, c.ok) |
| + } |
| + } |
| + // An hour an interval ENDS exactly at the start of is not touched. |
| + onHour := time.Date(2026, 9, 3, 16, 0, 0, 0, time.UTC) |
| + b2 := &BucketOwnership{V: 1, Bucket: ledgerBucket, Intervals: []OwnerInterval{{Account: "bab", App: "titogo", From: t0, Until: &onHour}}} |
| + if _, _, ok := b2.OwnerAt(onHour); ok { |
| + t.Error("an interval ending at 16:00 does not own the 16:00 hour") |
| + } |
| + var nilLedger *BucketOwnership |
| + if _, _, ok := nilLedger.OwnerAt(t0); ok { |
| + t.Error("no ledger, no owner") |
| + } |
| +} |
| + |
| +// Two owners in one hour is not an answer: the caller must hold. |
| +func TestOwnerAtRefusesSharedHour(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := CloseBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0.Add(time.Hour)); err != nil { |
| + t.Fatal(err) |
| + } |
| + // Another app picks the bucket up in the same hour the first let go. |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bac", "other", t0.Add(90*time.Minute)); err != nil { |
| + t.Fatal(err) |
| + } |
| + owners := BucketOwners{ledgerBucket: mustLedger(t, s, ledgerBucket)} |
| + if _, _, ok := owners.OwnerAt(ledgerBucket, time.Date(2026, 9, 3, 11, 0, 0, 0, time.UTC)); ok { |
| + t.Fatal("the 11:00 hour has two owners and must not resolve") |
| + } |
| + if acct, _, ok := owners.OwnerAt(ledgerBucket, time.Date(2026, 9, 3, 10, 0, 0, 0, time.UTC)); !ok || acct != "bab" { |
| + t.Fatal("the 10:00 hour belongs to bab alone") |
| + } |
| + if acct, _, ok := owners.OwnerAt(ledgerBucket, time.Date(2026, 9, 3, 13, 0, 0, 0, time.UTC)); !ok || acct != "bac" { |
| + t.Fatal("the 13:00 hour belongs to bac alone") |
| + } |
| + if _, _, ok := owners.OwnerAt("unknown-bucket", t0); ok { |
| + t.Fatal("a bucket with no ledger has no owner") |
| + } |
| + // Two apps opened concurrently on one adopted bucket: both stand, and |
| + // every shared hour is unresolved. |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0.Add(3*time.Hour)); err != nil { |
| + t.Fatal(err) |
| + } |
| + b := mustLedger(t, s, ledgerBucket) |
| + if len(b.Intervals) != 3 { |
| + t.Fatalf("intervals = %+v", b.Intervals) |
| + } |
| + if _, _, ok := b.OwnerAt(t0.Add(4 * time.Hour)); ok { |
| + t.Fatal("two open owners must not resolve") |
| + } |
| +} |
| + |
| +func TestAppendOnlyRefusesRewrites(t *testing.T) { |
| + end := t0.Add(time.Hour) |
| + prev := &BucketOwnership{V: 1, Bucket: ledgerBucket, Intervals: []OwnerInterval{ |
| + {Account: "bab", App: "titogo", From: t0, Until: &end}, |
| + {Account: "bab", App: "titogo", From: t0.Add(2 * time.Hour)}, |
| + }} |
| + later := end.Add(time.Hour) |
| + cases := map[string]*BucketOwnership{ |
| + "removed": {V: 1, Bucket: ledgerBucket, Intervals: prev.Intervals[:1]}, |
| + "moved from": {V: 1, Bucket: ledgerBucket, Intervals: []OwnerInterval{ |
| + {Account: "bab", App: "titogo", From: t0.Add(-time.Hour), Until: &end}, prev.Intervals[1]}}, |
| + "new owner": {V: 1, Bucket: ledgerBucket, Intervals: []OwnerInterval{ |
| + {Account: "bac", App: "titogo", From: t0, Until: &end}, prev.Intervals[1]}}, |
| + "extended close": {V: 1, Bucket: ledgerBucket, Intervals: []OwnerInterval{ |
| + {Account: "bab", App: "titogo", From: t0, Until: &later}, prev.Intervals[1]}}, |
| + "reopened": {V: 1, Bucket: ledgerBucket, Intervals: []OwnerInterval{ |
| + {Account: "bab", App: "titogo", From: t0}, prev.Intervals[1]}}, |
| + } |
| + for name, next := range cases { |
| + if err := appendOnly(prev, next); err == nil { |
| + t.Errorf("%s: accepted, want refused", name) |
| + } |
| + } |
| + closeOpen := &BucketOwnership{V: 1, Bucket: ledgerBucket, Intervals: []OwnerInterval{ |
| + prev.Intervals[0], {Account: "bab", App: "titogo", From: t0.Add(2 * time.Hour), Until: &later}}} |
| + if err := appendOnly(prev, closeOpen); err != nil { |
| + t.Errorf("closing an open interval must be allowed: %v", err) |
| + } |
| +} |
| + |
| +func TestReadBucketOwnershipRefusesBadLedgers(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + put := func(v any) { |
| + raw, _ := json.Marshal(v) |
| + s.Delete(ctx, BucketOwnershipKey(ledgerBucket)) |
| + if _, err := s.Put(ctx, BucketOwnershipKey(ledgerBucket), raw, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + before := t0.Add(-time.Hour) |
| + bad := map[string]any{ |
| + "wrong schema": BucketOwnership{V: 2, Bucket: ledgerBucket, Intervals: []OwnerInterval{{Account: "bab", App: "titogo", From: t0}}}, |
| + "wrong bucket": BucketOwnership{V: 1, Bucket: "other", Intervals: []OwnerInterval{{Account: "bab", App: "titogo", From: t0}}}, |
| + "no intervals": BucketOwnership{V: 1, Bucket: ledgerBucket}, |
| + "bad account": BucketOwnership{V: 1, Bucket: ledgerBucket, Intervals: []OwnerInterval{{Account: "../x", App: "titogo", From: t0}}}, |
| + "zero from": BucketOwnership{V: 1, Bucket: ledgerBucket, Intervals: []OwnerInterval{{Account: "bab", App: "titogo"}}}, |
| + "ends early": BucketOwnership{V: 1, Bucket: ledgerBucket, Intervals: []OwnerInterval{{Account: "bab", App: "titogo", From: t0, Until: &before}}}, |
| + "not even json": json.RawMessage(`"x"`), |
| + } |
| + for name, v := range bad { |
| + put(v) |
| + if _, _, err := ReadBucketOwnership(ctx, s, ledgerBucket); err == nil { |
| + t.Errorf("%s: read, want refused", name) |
| + } |
| + // A writer must refuse to build on a ledger it cannot read, never |
| + // overwrite it. |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0); err == nil { |
| + t.Errorf("%s: open wrote over an unreadable ledger", name) |
| + } |
| + } |
| + if _, _, err := ReadBucketOwnership(ctx, s, "Not_A_Bucket"); err == nil { |
| + t.Error("an invalid bucket name must be refused before any read") |
| + } |
| +} |
| + |
| +// racingOwnershipStore lands a competing write on the ledger just before |
| +// our first Put, so that Put loses its CAS. |
| +type racingOwnershipStore struct { |
| + release.Store |
| + raced bool |
| +} |
| + |
| +func (r *racingOwnershipStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| + if key == BucketOwnershipKey(ledgerBucket) && !r.raced { |
| + r.raced = true |
| + if err := OpenBucketOwner(ctx, r.Store, ledgerBucket, "bac", "other", t0.Add(-time.Hour)); err != nil { |
| + return "", err |
| + } |
| + } |
| + return r.Store.Put(ctx, key, body, ifMatch) |
| +} |
| + |
| +func TestOpenBucketOwnerRetriesLostCAS(t *testing.T) { |
| + ctx := context.Background() |
| + s := &racingOwnershipStore{Store: release.NewDirStore(t.TempDir())} |
| + if err := OpenBucketOwner(ctx, s, ledgerBucket, "bab", "titogo", t0); err != nil { |
| + t.Fatal(err) |
| + } |
| + b := mustLedger(t, s, ledgerBucket) |
| + if len(b.Intervals) != 2 || b.Intervals[0].Account != "bac" || b.Intervals[1].Account != "bab" { |
| + t.Fatalf("intervals = %+v, want the racing write kept and ours appended", b.Intervals) |
| + } |
| +} |
| + |
| +// Grant opens the interval from the grant block's own time, before any |
| +// credential exists; a re-grant leaves it alone; Teardown closes it only |
| +// after the bucket is gone. |
| +func TestGrantAndTeardownRecordOwnership(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + _, pub := testSealKeypair(t) |
| + p := NewMemProvisioner("eu-west-1") |
| + declare(t, s, "bab", "titogo", "TITO_S3") |
| + grantAt := time.Date(2026, 9, 1, 8, 0, 0, 0, time.UTC) |
| + if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", grantAt); err != nil { |
| + t.Fatal(err) |
| + } |
| + b := mustLedger(t, s, ledgerBucket) |
| + if len(b.Intervals) != 1 || !b.Intervals[0].From.Equal(grantAt) || b.Intervals[0].Until != nil || |
| + b.Intervals[0].Account != "bab" || b.Intervals[0].App != "titogo" { |
| + t.Fatalf("after grant: %+v", b.Intervals) |
| + } |
| + if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", grantAt.Add(24*time.Hour)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if b := mustLedger(t, s, ledgerBucket); len(b.Intervals) != 1 || !b.Intervals[0].From.Equal(grantAt) { |
| + t.Fatalf("after re-grant: %+v", b.Intervals) |
| + } |
| + |
| + closedAt := time.Date(2026, 9, 20, 12, 0, 0, 0, time.UTC) |
| + old := ownershipNow |
| + ownershipNow = func() time.Time { return closedAt } |
| + defer func() { ownershipNow = old }() |
| + if err := Teardown(ctx, s, p, nil, "bab", "titogo"); err != nil { |
| + t.Fatal(err) |
| + } |
| + b = mustLedger(t, s, ledgerBucket) |
| + if len(b.Intervals) != 1 || b.Intervals[0].Until == nil || !b.Intervals[0].Until.Equal(closedAt) { |
| + t.Fatalf("after teardown: %+v", b.Intervals) |
| + } |
| + // The ledger outlives the record: that is its whole point. |
| + if rec, _, _ := Get(ctx, s, "bab", "titogo"); rec != nil { |
| + t.Fatal("record should be gone after teardown") |
| + } |
| +} |
| + |
| +// A healthy store provisioned before the ledger existed is backfilled from |
| +// its grant time the next time Grant walks it. |
| +func TestGrantBackfillsLedgerForHealthyStore(t *testing.T) { |
| + ctx := context.Background() |
| + s := release.NewDirStore(t.TempDir()) |
| + _, pub := testSealKeypair(t) |
| + p := NewMemProvisioner("eu-west-1") |
| + declare(t, s, "bab", "titogo", "TITO_S3") |
| + grantAt := time.Date(2026, 8, 1, 8, 0, 0, 0, time.UTC) |
| + if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", grantAt); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := s.Delete(ctx, BucketOwnershipKey(ledgerBucket)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", grantAt.AddDate(0, 1, 0)); err != nil { |
| + t.Fatal(err) |
| + } |
| + b := mustLedger(t, s, ledgerBucket) |
| + if len(b.Intervals) != 1 || !b.Intervals[0].From.Equal(grantAt) { |
| + t.Fatalf("backfill: %+v, want from the grant block's time %s", b.Intervals, grantAt) |
| + } |
| +} |
| + |
| +// A ledger write failure stops Grant before any credential is minted. |
| +func TestGrantStopsWhenLedgerWriteFails(t *testing.T) { |
| + ctx := context.Background() |
| + s := &failingStore{Store: release.NewDirStore(t.TempDir()), failKeyOnce: BucketOwnershipKey(ledgerBucket)} |
| + _, pub := testSealKeypair(t) |
| + p := NewMemProvisioner("eu-west-1") |
| + declare(t, s, "bab", "titogo", "TITO_S3") |
| + _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", t0) |
| + if err == nil || errors.Is(err, ErrNotDeclared) { |
| + t.Fatalf("err = %v, want the injected ledger failure", err) |
| + } |
| + if len(p.Buckets) != 0 { |
| + t.Fatalf("buckets = %v: nothing may be created before ownership is recorded", p.Buckets) |
| + } |
| + for _, u := range p.Users { |
| + if len(u.Keys) != 0 { |
| + t.Fatalf("a key was minted before ownership was recorded") |
| + } |
| + } |
| + // Re-running repairs. |
| + if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", t0); err != nil { |
| + t.Fatal(err) |
| + } |
| + mustLedger(t, s, ledgerBucket) |
| +} |
diff --git a/internal/objstore/walk.go b/internal/objstore/walk.go| index dd50dc0c2..c50f6c8c0 100644 |
| --- a/internal/objstore/walk.go |
| +++ b/internal/objstore/walk.go |
| @@ -172,6 +172,16 @@ func Grant(ctx context.Context, s release.Store, p Provisioner, d Deployment, |
| return nil, err |
| } |
| } |
| + // 1b. bucket ownership history (ownership.go). Opened here, before the |
| + // idempotence short-circuit and before any credential exists, so that |
| + // no tenant can move a byte out of this bucket while the ledger names |
| + // nobody, and so that a healthy store re-granted by an operator is |
| + // backfilled from its own grant time. Idempotent: an open interval for |
| + // this app is left untouched. A failure stops the walk before anything |
| + // is created; re-running it repairs. |
| + if err := OpenBucketOwner(ctx, s, bucket, acct, app, grantOwnershipFrom(rec.Grant, now)); err != nil { |
| + return nil, fmt.Errorf("objstore: record ownership of %s: %w", bucket, err) |
| + } |
| // 2. the principal and its policy. EnsureUser runs UNCONDITIONALLY, before |
| // the idempotence short-circuit below — the same fix internal/mail's walk |
| // made as "C2", for the same reason. |
| @@ -532,6 +542,17 @@ func Teardown(ctx context.Context, s release.Store, p Provisioner, sc Scanner, a |
| if err := p.DeleteBucket(ctx, bucket); err != nil { |
| return fmt.Errorf("objstore: delete bucket %s: %w", bucket, err) |
| } |
| + // Close this app's ownership interval (ownership.go) only now that |
| + // the bucket is gone, so no byte can leave it after the ledger |
| + // stops naming the app. Before the record goes, so a failure here |
| + // leaves the record standing and a re-run closes it. A name Grant |
| + // would refuse can never have had an interval opened, so there is |
| + // nothing to close for one. |
| + if ValidBucketName(bucket) { |
| + if err := CloseBucketOwner(ctx, s, bucket, acct, app, ownershipNow()); err != nil { |
| + return fmt.Errorf("objstore: close ownership of %s: %w", bucket, err) |
| + } |
| + } |
| } |
| // DeleteUser reaps the keys and the inline policy, so the credential dies |
| // with the estate it was scoped to — no orphan principal keeps a key that |