amadan
release: PurgeApp refuses a provisioned or granted object store
Box-side auto-purge must never sweep away the only record of a paid bucket and a live IAM user: boxes hold no backend authority, so the console's purge door tears the store down first and only then re-enters PurgeApp. Both witnesses refuse. `provisioned` is the finished store; `grant` on its own is a grant that crashed after creating the bucket and the user but before recording them -- the earliest bucket-visible sign that estate exists. Unreadable refuses too, transport error or unparseable bytes alike, the same direction as the production flag: "I could not ask" is not "no store". release cannot import objstore (objstore imports release), so the two fields are parsed in place with a local struct -- which puts release on store.json's parser list, noted on both sides. objstore's TestPurgeGuardSeesOurRecord drives real Grant/crashed-grant records through release.PurgeApp so a renamed key fails the build's tests rather than silently disarming a fleet-wide guard.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
4 files changed,
+246
−0
internal/objstore/objstore.go+9 −0internal/objstore/walk_test.go+71 −0internal/release/trash.go+42 −0internal/release/trash_test.go+124 −0
diff --git a/internal/objstore/objstore.go b/internal/objstore/objstore.go| index 9458c4d94..8478248f0 100644 |
| --- a/internal/objstore/objstore.go |
| +++ b/internal/objstore/objstore.go |
| @@ -50,6 +50,15 @@ type Provisioned struct { |
| // Record is apps/<acct>/<app>/store.json — three blocks for three actors |
| // (member declares, operator grants, console provisions). The secret key is |
| // never here; it exists only sealed inside config.json. |
| +// |
| +// PARSER LIST: this package is not the only reader of this shape. |
| +// release.PurgeApp parses `grant` and `provisioned` out of these bytes with |
| +// its own local struct — release cannot import objstore, because objstore |
| +// imports release — and REFUSES the box-side auto-purge when either block is |
| +// present. Rename or restructure either key here and that fleet-wide guard |
| +// silently stops seeing backend estate; the mirror test in walk_test.go |
| +// (TestPurgeGuardSeesOurRecord) exists to fail loudly if that ever happens. |
| +// Additive changes are safe; these two names are not free to move. |
| type Record struct { |
| Schema int `json:"schema"` |
| App string `json:"app"` |
diff --git a/internal/objstore/walk_test.go b/internal/objstore/walk_test.go| index 95f194ca4..1bd07b52d 100644 |
| --- a/internal/objstore/walk_test.go |
| +++ b/internal/objstore/walk_test.go |
| @@ -3,6 +3,8 @@ package objstore |
| import ( |
| "context" |
| "crypto/ecdh" |
| + "crypto/ed25519" |
| + "crypto/rand" |
| "errors" |
| "os" |
| "strings" |
| @@ -1292,3 +1294,72 @@ func TestTeardownRefusesConcurrentRegrant(t *testing.T) { |
| t.Fatalf("the competing grant's record was erased: %+v, %v", stored, err) |
| } |
| } |
| + |
| +// TestPurgeGuardSeesOurRecord pins the PARSE AGREEMENT between this package |
| +// and release.PurgeApp. release cannot import objstore (objstore imports |
| +// release), so PurgeApp re-parses `grant` and `provisioned` out of store.json |
| +// with a local struct of its own. That guard is what stops a box's auto-purge |
| +// from sweeping away the only record of a paid bucket and a live IAM user. |
| +// Records written HERE, by the real writer, must trip it — for the finished |
| +// store and for the crashed grant alike. |
| +func TestPurgeGuardSeesOurRecord(t *testing.T) { |
| + ctx := context.Background() |
| + _, pub := testSealKeypair(t) |
| + relPub, _, err := ed25519.GenerateKey(rand.Reader) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + // Past the cooloff, so nothing but the store guard stands between the |
| + // purge and destruction. |
| + backdated := time.Now().Add(-2 * release.TrashCooloff) |
| + |
| + t.Run("provisioned", func(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + p := NewMemProvisioner("eu-west-1") |
| + rec := rotateFixture(t, s, pub, p) // declare + Grant: a real provisioned record |
| + if rec.Provisioned == nil { |
| + t.Fatal("fixture produced no provisioned block") |
| + } |
| + if err := release.TrashApp(ctx, s, "bab", "titogo", "paul@keymail.dev", backdated); err != nil { |
| + t.Fatal(err) |
| + } |
| + err := release.PurgeApp(ctx, s, relPub, "bab", "titogo", time.Now()) |
| + if !errors.Is(err, release.ErrStoreProvisioned) { |
| + t.Fatalf("PurgeApp over a provisioned record = %v, want release.ErrStoreProvisioned", err) |
| + } |
| + if _, _, err := s.Get(ctx, Key("bab", "titogo")); err != nil { |
| + t.Fatalf("the refused purge destroyed store.json: %v", err) |
| + } |
| + }) |
| + |
| + t.Run("crashed-grant", func(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + p := NewMemProvisioner("eu-west-1") |
| + crashedGrant(t, s, p) // grant block, no provisioned block, live estate |
| + if err := release.TrashApp(ctx, s, "bab", "titogo", "paul@keymail.dev", backdated); err != nil { |
| + t.Fatal(err) |
| + } |
| + err := release.PurgeApp(ctx, s, relPub, "bab", "titogo", time.Now()) |
| + if !errors.Is(err, release.ErrStoreProvisioned) { |
| + t.Fatalf("PurgeApp over a crashed grant = %v, want release.ErrStoreProvisioned", err) |
| + } |
| + }) |
| + |
| + // The other direction, so the guard cannot pass by refusing everything: a |
| + // declared-only record has no backend estate and is purged like any other |
| + // object. This is also the state Teardown leaves behind (it deletes the |
| + // record outright), which is how a torn-down app becomes purgeable. |
| + t.Run("declared-only-purges", func(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + declare(t, s, "bab", "titogo", "TITO_S3") |
| + if err := release.TrashApp(ctx, s, "bab", "titogo", "paul@keymail.dev", backdated); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := release.PurgeApp(ctx, s, relPub, "bab", "titogo", time.Now()); err != nil { |
| + t.Fatalf("PurgeApp over a declared-only record: %v", err) |
| + } |
| + if _, _, err := s.Get(ctx, Key("bab", "titogo")); !errors.Is(err, os.ErrNotExist) { |
| + t.Fatalf("store.json survived the purge: %v", err) |
| + } |
| + }) |
| +} |
diff --git a/internal/release/trash.go b/internal/release/trash.go| index c5288a07c..0ba27cb34 100644 |
| --- a/internal/release/trash.go |
| +++ b/internal/release/trash.go |
| @@ -105,6 +105,11 @@ func RestoreApp(ctx context.Context, s Store, acct, app string) error { |
| // "already done", not as a failure worth alarming about. |
| var ErrNotTrashed = errors.New("release: app is not in the trash") |
| +// ErrStoreProvisioned is PurgeApp's refusal to destroy an app whose object |
| +// store has (or may have) backend estate behind it — see the guard in |
| +// PurgeApp for why a box is the wrong actor to decide otherwise. |
| +var ErrStoreProvisioned = errors.New("release: app has a provisioned object store — tear it down via the console before purging") |
| + |
| // PurgeApp irreversibly destroys a trashed app's bucket objects: every key |
| // under apps/<acct>/<app>/, the trash marker LAST, so a purge that dies |
| // half-way leaves a trashed app with fewer objects rather than a live app |
| @@ -148,6 +153,43 @@ func PurgeApp(ctx context.Context, s Store, pub ed25519.PublicKey, acct, app str |
| return fmt.Errorf("release: %s/%s is flagged production — a trashed production app should be impossible; refusing to purge", acct, app) |
| } |
| prefix := appPrefix(acct, app) |
| + // An object store with backend estate refuses the purge outright (objstore |
| + // spec §9): auto-purge runs on BOXES, boxes hold no backend authority, and |
| + // a swept store.json is a paid bucket and a live IAM user that nothing |
| + // remembers. The console's purge door tears the store down first and only |
| + // then re-enters here. |
| + // |
| + // BOTH blocks count. `provisioned` is the finished store; `grant` on its |
| + // own is a grant that crashed after creating the bucket and the user but |
| + // before recording them — the earliest bucket-visible witness that estate |
| + // exists. Refusing on either keeps the box on the safe side of a crash. |
| + // |
| + // Parsed minimally IN PLACE: release cannot import objstore (objstore |
| + // imports release), so this file is on store.json's parser list, |
| + // deliberately — the consumer list for a storage shape is everything that |
| + // PARSES it, not everything that calls its API. objstore.Record carries |
| + // the matching note; a block renamed there without changing this struct |
| + // silently disarms the guard. objstore/walk_test.go pins the agreement. |
| + // |
| + // Unreadable — transport error or bytes that will not parse — refuses too, |
| + // the same direction as the production flag: "I could not ask" is not "no |
| + // store". Absent is fine; most apps never declare one. |
| + if raw, _, err := s.Get(ctx, prefix+"store.json"); err == nil { |
| + var rec struct { |
| + Grant *json.RawMessage `json:"grant"` |
| + Provisioned *json.RawMessage `json:"provisioned"` |
| + } |
| + if jerr := json.Unmarshal(raw, &rec); jerr != nil { |
| + return fmt.Errorf("release: %s/%s: unparseable %sstore.json — refusing to purge: %w: %v", |
| + acct, app, prefix, ErrStoreProvisioned, jerr) |
| + } |
| + if rec.Provisioned != nil || rec.Grant != nil { |
| + return fmt.Errorf("%s/%s: %w", acct, app, ErrStoreProvisioned) |
| + } |
| + } else if !errors.Is(err, os.ErrNotExist) { |
| + return fmt.Errorf("release: %s/%s: cannot read %sstore.json — refusing to purge: %w: %v", |
| + acct, app, prefix, ErrStoreProvisioned, err) |
| + } |
| marker := AppTombstoneKey(acct, app) |
| keys, err := s.List(ctx, prefix) |
| if err != nil { |
diff --git a/internal/release/trash_test.go b/internal/release/trash_test.go| index 8048630d1..43794d7b3 100644 |
| --- a/internal/release/trash_test.go |
| +++ b/internal/release/trash_test.go |
| @@ -4,6 +4,7 @@ import ( |
| "context" |
| "crypto/ed25519" |
| "errors" |
| + "os" |
| "slices" |
| "strings" |
| "testing" |
| @@ -345,3 +346,126 @@ func TestPurgeAppIsIdempotent(t *testing.T) { |
| t.Fatalf("second purge = %v, want ErrNotTrashed", err) |
| } |
| } |
| + |
| +// storeJSON writes apps/<acct>/<app>/store.json, overwriting whatever is |
| +// there — the purge guard's input, hand-rolled here because release cannot |
| +// import objstore (objstore imports release). The bytes ARE the contract. |
| +func storeJSON(t *testing.T, s Store, acct, app, body string) { |
| + t.Helper() |
| + key := "apps/" + acct + "/" + app + "/store.json" |
| + _, etag, err := s.Get(ctx, key) |
| + if err != nil && !errors.Is(err, os.ErrNotExist) { |
| + t.Fatal(err) |
| + } |
| + if _, err := s.Put(ctx, key, []byte(body), etag); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +// Contract 6: a store with backend estate refuses the purge. Auto-purge runs |
| +// on boxes; boxes hold no backend authority, so a swept store.json is a paid |
| +// bucket nothing remembers. BOTH witnesses count: `provisioned` (the finished |
| +// store) and `grant` (a grant that may have crashed after creating the bucket |
| +// and the IAM user but before recording them). A declared-only record has no |
| +// estate behind it and is swept like any other object. |
| +func TestPurgeAppRefusesProvisionedOrGrantedStore(t *testing.T) { |
| + _, pub := testKeys(t) |
| + |
| + for _, tc := range []struct { |
| + name string |
| + body string |
| + }{ |
| + {"provisioned", `{"schema":1,"app":"derp","declared":{"env_prefix":"CARLOS_STORE"},"provisioned":{"bucket":"carlos-store-tito-bab-derp","key_id":"AKIA"}}`}, |
| + // A grant block with NO provisioned block: the earliest bucket-visible |
| + // witness that backend estate may already exist. |
| + {"grant-only", `{"schema":1,"app":"derp","declared":{"env_prefix":"CARLOS_STORE"},"grant":{"backend":"aws","by":"paul@keymail.dev"}}`}, |
| + // Unreadable is not "no store" — same direction as the production flag. |
| + {"unparseable", `{"schema":1,`}, |
| + } { |
| + t.Run(tc.name, func(t *testing.T) { |
| + s := NewDirStore(t.TempDir()) |
| + keys := seedApp(t, s, "bab", "derp") |
| + storeJSON(t, s, "bab", "derp", tc.body) |
| + if err := TrashApp(ctx, s, "bab", "derp", "paul@keymail.dev", time.Now().Add(-31*24*time.Hour)); err != nil { |
| + t.Fatal(err) |
| + } |
| + err := PurgeApp(ctx, s, pub, "bab", "derp", time.Now()) |
| + if !errors.Is(err, ErrStoreProvisioned) { |
| + t.Fatalf("PurgeApp = %v, want ErrStoreProvisioned", err) |
| + } |
| + // The refusal names the app: one line in the auto-purge log has to |
| + // be enough to know which store needs tearing down. |
| + if !strings.Contains(err.Error(), "bab/derp") { |
| + t.Errorf("refusal %q does not name the app", err) |
| + } |
| + if got := listUnder(t, s, "apps/bab/derp/"); len(got) != len(keys)+2 { |
| + t.Fatalf("a refused purge destroyed objects: %v", got) |
| + } |
| + }) |
| + } |
| + |
| + // Declared-only: no backend estate exists, so the purge proceeds and |
| + // sweeps store.json along with everything else. |
| + t.Run("declared-only", func(t *testing.T) { |
| + s := NewDirStore(t.TempDir()) |
| + seedApp(t, s, "bab", "derp") |
| + storeJSON(t, s, "bab", "derp", `{"schema":1,"app":"derp","declared":{"env_prefix":"CARLOS_STORE"}}`) |
| + if err := TrashApp(ctx, s, "bab", "derp", "paul@keymail.dev", time.Now().Add(-31*24*time.Hour)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := PurgeApp(ctx, s, pub, "bab", "derp", time.Now()); err != nil { |
| + t.Fatalf("PurgeApp of a declared-only store: %v", err) |
| + } |
| + if got := listUnder(t, s, "apps/bab/derp/"); len(got) != 0 { |
| + t.Fatalf("purge left objects behind: %v", got) |
| + } |
| + }) |
| + |
| + // An explicitly null block is absent, not present: a record that has been |
| + // torn down keeps its keys with null values and must not wedge the purge. |
| + t.Run("null-blocks", func(t *testing.T) { |
| + s := NewDirStore(t.TempDir()) |
| + seedApp(t, s, "bab", "derp") |
| + storeJSON(t, s, "bab", "derp", `{"schema":1,"app":"derp","grant":null,"provisioned":null}`) |
| + if err := TrashApp(ctx, s, "bab", "derp", "paul@keymail.dev", time.Now().Add(-31*24*time.Hour)); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := PurgeApp(ctx, s, pub, "bab", "derp", time.Now()); err != nil { |
| + t.Fatalf("PurgeApp of a torn-down store: %v", err) |
| + } |
| + }) |
| +} |
| + |
| +// erroringGetStore fails Get for one key with something that is NOT |
| +// os.ErrNotExist — the transport-level "I cannot tell you whether this app |
| +// has a store" the guard must refuse on. |
| +type erroringGetStore struct { |
| + Store |
| + failKey string |
| +} |
| + |
| +func (e erroringGetStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + if key == e.failKey { |
| + return nil, "", errors.New("injected: get failed") |
| + } |
| + return e.Store.Get(ctx, key) |
| +} |
| + |
| +// Contract 6b: a store record the box cannot READ refuses too. "I could not |
| +// ask" is not "there is no store". |
| +func TestPurgeAppRefusesUnreadableStoreRecord(t *testing.T) { |
| + _, pub := testKeys(t) |
| + base := NewDirStore(t.TempDir()) |
| + s := erroringGetStore{Store: base, failKey: "apps/bab/derp/store.json"} |
| + keys := seedApp(t, base, "bab", "derp") |
| + if err := TrashApp(ctx, base, "bab", "derp", "paul@keymail.dev", time.Now().Add(-31*24*time.Hour)); err != nil { |
| + t.Fatal(err) |
| + } |
| + err := PurgeApp(ctx, s, pub, "bab", "derp", time.Now()) |
| + if err == nil { |
| + t.Fatal("purge acted on an unreadable store record") |
| + } |
| + if got := listUnder(t, base, "apps/bab/derp/"); len(got) != len(keys)+1 { |
| + t.Fatalf("a refused purge destroyed objects: %v", got) |
| + } |
| +} |