amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package objstore
2
3import (
4 "context"
5 "crypto/ecdh"
6 "crypto/ed25519"
7 "crypto/rand"
8 "errors"
9 "os"
10 "strings"
11 "testing"
12 "time"
13
14 "github.com/carlosframework/platform/internal/appconfig"
15 "github.com/carlosframework/platform/internal/release"
16)
17
18// testSealKeypair mints the secrets keypair the walk seals with and the test
19// opens with — the console holds the public half, a host the private one.
20func testSealKeypair(t *testing.T) (*ecdh.PrivateKey, *ecdh.PublicKey) {
21 t.Helper()
22 privB64, pubB64, err := appconfig.GenerateKey()
23 if err != nil {
24 t.Fatal(err)
25 }
26 priv, err := appconfig.ParsePrivateKey(privB64, "test")
27 if err != nil {
28 t.Fatal(err)
29 }
30 pub, err := appconfig.ParsePublicKey(pubB64, "test")
31 if err != nil {
32 t.Fatal(err)
33 }
34 return priv, pub
35}
36
37// failingStore fails the FIRST Put to failKeyOnce, then behaves normally —
38// the crash between mint and delivery the walk has to repair.
39type failingStore struct {
40 release.Store
41 failKeyOnce string
42}
43
44func (f *failingStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) {
45 if key != "" && key == f.failKeyOnce {
46 f.failKeyOnce = ""
47 return "", errors.New("injected: put failed")
48 }
49 return f.Store.Put(ctx, key, body, ifMatch)
50}
51
52func testDeployment() Deployment {
53 return Deployment{Backend: "aws", Region: "eu-west-1", BucketPrefix: "carlos-store-tito-"}
54}
55
56// declare writes the member half of the record: a store is asked for.
57func declare(t *testing.T, s release.Store, acct, app, prefix string) {
58 t.Helper()
59 rec := &Record{Declared: &Declared{EnvPrefix: prefix, At: time.Now().UTC(), By: "member@example.com"}}
60 if _, err := Put(context.Background(), s, acct, app, rec, ""); err != nil {
61 t.Fatal(err)
62 }
63}
64
65func mustVars(t *testing.T, s release.Store, acct, app string) map[string]appconfig.Var {
66 t.Helper()
67 m, err := appconfig.Get(context.Background(), s, acct, app)
68 if err != nil {
69 t.Fatal(err)
70 }
71 return m.Vars
72}
73
74func TestGrantUndeclaredRefused(t *testing.T) {
75 ctx := context.Background()
76 s := release.NewDirStore(t.TempDir())
77 _, pub := testSealKeypair(t)
78 p := NewMemProvisioner("eu-west-1")
79 if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC()); !errors.Is(err, ErrNotDeclared) {
80 t.Fatalf("Grant on undeclared app: %v, want ErrNotDeclared", err)
81 }
82 if len(p.Buckets) != 0 || len(p.Users) != 0 {
83 t.Fatalf("refused grant touched the backend: %v %v", p.Buckets, p.Users)
84 }
85}
86
87func TestGrantMintsAndDelivers(t *testing.T) {
88 ctx := context.Background()
89 s := release.NewDirStore(t.TempDir())
90 priv, pub := testSealKeypair(t)
91 p := NewMemProvisioner("eu-west-1")
92 declare(t, s, "bab", "titogo", "TITO_S3")
93
94 now := time.Now().UTC()
95 rec, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op@example.com", GrantSourceOperator, "", now)
96 if err != nil {
97 t.Fatal(err)
98 }
99
100 const wantBucket = "carlos-store-tito-bab-titogo"
101 const wantUser = "carlos-store-bab-titogo"
102 if _, ok := p.Buckets[wantBucket]; !ok {
103 t.Fatalf("bucket %q not created: %v", wantBucket, p.Buckets)
104 }
105 u, ok := p.Users[wantUser]
106 if !ok {
107 t.Fatalf("user %q not created: %v", wantUser, p.Users)
108 }
109 if rec.Provisioned == nil {
110 t.Fatal("record has no provisioned block")
111 }
112 if rec.Provisioned.Bucket != wantBucket || rec.Provisioned.IAMUser != wantUser ||
113 rec.Provisioned.Region != "eu-west-1" || rec.Provisioned.UserARN != u.ARN {
114 t.Fatalf("provisioned = %+v", rec.Provisioned)
115 }
116 if rec.Grant == nil || rec.Grant.Backend != "aws" || rec.Grant.By != "op@example.com" {
117 t.Fatalf("grant block = %+v", rec.Grant)
118 }
119 secret, ok := u.Keys[rec.Provisioned.KeyID]
120 if !ok {
121 t.Fatalf("recorded key %q is not in the fake: %v", rec.Provisioned.KeyID, u.Keys)
122 }
123
124 // the record is durable, not just the returned value
125 stored, _, err := Get(ctx, s, "bab", "titogo")
126 if err != nil || stored == nil || stored.Provisioned == nil ||
127 stored.Provisioned.KeyID != rec.Provisioned.KeyID {
128 t.Fatalf("stored record = %+v, %v", stored, err)
129 }
130
131 vars := mustVars(t, s, "bab", "titogo")
132 if v := vars["TITO_S3_BUCKET"]; v.Kind != "env" || v.Value != wantBucket {
133 t.Fatalf("TITO_S3_BUCKET = %+v", v)
134 }
135 if v := vars["TITO_S3_REGION"]; v.Kind != "env" || v.Value != "eu-west-1" {
136 t.Fatalf("TITO_S3_REGION = %+v", v)
137 }
138 if v := vars["TITO_S3_ACCESS_KEY"]; v.Kind != "secret" || v.Envelope == "" {
139 t.Fatalf("TITO_S3_ACCESS_KEY = %+v", v)
140 }
141 gotID, err := appconfig.Open(priv, appconfig.Binding{Account: "bab", App: "titogo"}, vars["TITO_S3_ACCESS_KEY"].Envelope)
142 if err != nil || string(gotID) != rec.Provisioned.KeyID {
143 t.Fatalf("delivered access key = %q, %v; want %q", gotID, err, rec.Provisioned.KeyID)
144 }
145 sk := vars["TITO_S3_SECRET_KEY"]
146 if sk.Kind != "secret" || sk.Value != "" {
147 t.Fatalf("TITO_S3_SECRET_KEY = %+v (secret must never be plaintext)", sk)
148 }
149 gotSecret, err := appconfig.Open(priv, appconfig.Binding{Account: "bab", App: "titogo"}, sk.Envelope)
150 if err != nil || string(gotSecret) != secret {
151 t.Fatalf("delivered secret = %q, %v; want %q", gotSecret, err, secret)
152 }
153 if _, ok := vars["TITO_S3_ENDPOINT"]; ok {
154 t.Fatalf("endpoint delivered for an empty endpoint: %+v", vars)
155 }
156}
157
158func TestGrantDeliversEndpointWhenSet(t *testing.T) {
159 ctx := context.Background()
160 s := release.NewDirStore(t.TempDir())
161 _, pub := testSealKeypair(t)
162 p := NewMemProvisioner("eu-central-1")
163 declare(t, s, "bab", "titogo", "TITO_S3")
164
165 d := testDeployment()
166 d.Endpoint = "https://s3.example.net"
167 rec, err := Grant(ctx, s, p, d, pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC())
168 if err != nil {
169 t.Fatal(err)
170 }
171 if rec.Provisioned.Endpoint != d.Endpoint {
172 t.Fatalf("provisioned endpoint = %q", rec.Provisioned.Endpoint)
173 }
174 vars := mustVars(t, s, "bab", "titogo")
175 if v := vars["TITO_S3_ENDPOINT"]; v.Kind != "env" || v.Value != d.Endpoint {
176 t.Fatalf("TITO_S3_ENDPOINT = %+v", v)
177 }
178}
179
180func TestGrantAdoptsExistingBucket(t *testing.T) {
181 ctx := context.Background()
182 s := release.NewDirStore(t.TempDir())
183 _, pub := testSealKeypair(t)
184 p := NewMemProvisioner("eu-west-1")
185 declare(t, s, "bab", "titogo", "TITO_S3")
186
187 rec, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "titogo-uploads", time.Now().UTC())
188 if err != nil {
189 t.Fatal(err)
190 }
191 if rec.Provisioned.Bucket != "titogo-uploads" || rec.Grant.Bucket != "titogo-uploads" {
192 t.Fatalf("adopt: grant=%+v provisioned=%+v", rec.Grant, rec.Provisioned)
193 }
194 if _, ok := p.Buckets["carlos-store-tito-bab-titogo"]; ok {
195 t.Fatalf("minted bucket created despite adoption: %v", p.Buckets)
196 }
197 if v := mustVars(t, s, "bab", "titogo")["TITO_S3_BUCKET"]; v.Value != "titogo-uploads" {
198 t.Fatalf("delivered bucket = %+v", v)
199 }
200}
201
202func TestGrantResumesAfterEachStepFailure(t *testing.T) {
203 for _, method := range []string{"EnsureBucket", "EnsureUser", "MintKey"} {
204 t.Run(method, func(t *testing.T) {
205 ctx := context.Background()
206 s := release.NewDirStore(t.TempDir())
207 priv, pub := testSealKeypair(t)
208 p := NewMemProvisioner("eu-west-1")
209 declare(t, s, "bab", "titogo", "TITO_S3")
210
211 p.FailNext[method] = errors.New("injected: " + method + " failed")
212 if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC()); err == nil {
213 t.Fatalf("%s failure: Grant returned no error", method)
214 }
215 // The grant block is written before any backend call, so a
216 // half-failed walk leaves the bucket choice pinned, not provisioned.
217 half, _, err := Get(ctx, s, "bab", "titogo")
218 if err != nil || half == nil || half.Grant == nil || half.Provisioned != nil {
219 t.Fatalf("record after %s failure = %+v, %v", method, half, err)
220 }
221
222 rec, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC())
223 if err != nil {
224 t.Fatalf("resume after %s failure: %v", method, err)
225 }
226 if len(p.Buckets) != 1 {
227 t.Fatalf("buckets after resume = %v", p.Buckets)
228 }
229 if len(p.Users) != 1 {
230 t.Fatalf("users after resume = %v", p.Users)
231 }
232 u := p.Users["carlos-store-bab-titogo"]
233 if len(u.Keys) != 1 {
234 t.Fatalf("keys after resume = %v", u.Keys)
235 }
236 if _, ok := u.Keys[rec.Provisioned.KeyID]; !ok {
237 t.Fatalf("recorded key %q not live: %v", rec.Provisioned.KeyID, u.Keys)
238 }
239 env := mustVars(t, s, "bab", "titogo")["TITO_S3_SECRET_KEY"].Envelope
240 got, err := appconfig.Open(priv, appconfig.Binding{Account: "bab", App: "titogo"}, env)
241 if err != nil || string(got) != u.Keys[rec.Provisioned.KeyID] {
242 t.Fatalf("delivered secret = %q, %v; want %q", got, err, u.Keys[rec.Provisioned.KeyID])
243 }
244 })
245 }
246
247 t.Run("delivery", func(t *testing.T) {
248 ctx := context.Background()
249 priv, pub := testSealKeypair(t)
250 s := &failingStore{Store: release.NewDirStore(t.TempDir()), failKeyOnce: appconfig.Key("bab", "titogo")}
251 p := NewMemProvisioner("eu-west-1")
252 declare(t, s, "bab", "titogo", "TITO_S3")
253
254 if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC()); err == nil {
255 t.Fatal("delivery failure: Grant returned no error")
256 }
257 // The record must NOT claim provisioned: the secret never landed.
258 if rec, _, err := Get(ctx, s, "bab", "titogo"); err != nil || rec == nil || rec.Provisioned != nil {
259 t.Fatalf("record after failed delivery = %+v, %v", rec, err)
260 }
261
262 rec, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC())
263 if err != nil {
264 t.Fatalf("resume after delivery failure: %v", err)
265 }
266 u := p.Users["carlos-store-bab-titogo"]
267 if len(u.Keys) != 1 {
268 t.Fatalf("undelivered key not reaped: %v", u.Keys)
269 }
270 if rec.Provisioned.KeyID != "MEMKEY2" {
271 t.Fatalf("recorded key = %q, want the SECOND mint MEMKEY2", rec.Provisioned.KeyID)
272 }
273 vars := mustVars(t, s, "bab", "titogo")
274 gotID, err := appconfig.Open(priv, appconfig.Binding{Account: "bab", App: "titogo"}, vars["TITO_S3_ACCESS_KEY"].Envelope)
275 if err != nil || string(gotID) != "MEMKEY2" {
276 t.Fatalf("delivered access key = %q, %v; want MEMKEY2", gotID, err)
277 }
278 gotSecret, err := appconfig.Open(priv, appconfig.Binding{Account: "bab", App: "titogo"}, vars["TITO_S3_SECRET_KEY"].Envelope)
279 if err != nil || string(gotSecret) != u.Keys["MEMKEY2"] {
280 t.Fatalf("delivered secret = %q, %v; want %q", gotSecret, err, u.Keys["MEMKEY2"])
281 }
282 })
283}
284
285func TestGrantRerunKeepsBucketChoice(t *testing.T) {
286 ctx := context.Background()
287 s := release.NewDirStore(t.TempDir())
288 _, pub := testSealKeypair(t)
289 p := NewMemProvisioner("eu-west-1")
290 declare(t, s, "bab", "titogo", "TITO_S3")
291
292 if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "titogo-uploads", time.Now().UTC()); err != nil {
293 t.Fatal(err)
294 }
295 // Wipe delivery so the re-run walks the whole sequence again.
296 if _, err := appconfig.Apply(ctx, s, "bab", "titogo", nil, []string{"TITO_S3_BUCKET"}); err != nil {
297 t.Fatal(err)
298 }
299 rec, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC())
300 if err != nil {
301 t.Fatal(err)
302 }
303 if rec.Provisioned.Bucket != "titogo-uploads" || rec.Grant.Bucket != "titogo-uploads" {
304 t.Fatalf("re-run re-chose the bucket: grant=%+v provisioned=%+v", rec.Grant, rec.Provisioned)
305 }
306 if len(p.Buckets) != 1 {
307 t.Fatalf("buckets = %v", p.Buckets)
308 }
309 if v := mustVars(t, s, "bab", "titogo")["TITO_S3_BUCKET"]; v.Value != "titogo-uploads" {
310 t.Fatalf("re-delivered bucket = %+v", v)
311 }
312}
313
314func TestGrantIdempotentWhenHealthy(t *testing.T) {
315 ctx := context.Background()
316 s := release.NewDirStore(t.TempDir())
317 _, pub := testSealKeypair(t)
318 p := NewMemProvisioner("eu-west-1")
319 declare(t, s, "bab", "titogo", "TITO_S3")
320
321 first, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC())
322 if err != nil {
323 t.Fatal(err)
324 }
325 before := mustVars(t, s, "bab", "titogo")
326
327 second, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC().Add(time.Hour))
328 if err != nil {
329 t.Fatal(err)
330 }
331 if second.Provisioned.KeyID != first.Provisioned.KeyID {
332 t.Fatalf("re-run minted a fresh key: %q then %q", first.Provisioned.KeyID, second.Provisioned.KeyID)
333 }
334 if u := p.Users["carlos-store-bab-titogo"]; len(u.Keys) != 1 {
335 t.Fatalf("keys after healthy re-run = %v", u.Keys)
336 }
337 after := mustVars(t, s, "bab", "titogo")
338 if len(before) != len(after) {
339 t.Fatalf("var set changed: %v then %v", before, after)
340 }
341 for k, v := range before {
342 if after[k] != v {
343 t.Fatalf("var %s rewritten: %+v then %+v", k, v, after[k])
344 }
345 }
346}
347
348// A member (or an older tool) can occupy a delivered NAME with the wrong
349// KIND — a key half stored as a plaintext env var is the motivating case. The
350// app cannot use it, so the store is not delivered and the walk must repair.
351func TestGrantRepairsWrongKindVar(t *testing.T) {
352 ctx := context.Background()
353 s := release.NewDirStore(t.TempDir())
354 priv, pub := testSealKeypair(t)
355 p := NewMemProvisioner("eu-west-1")
356 declare(t, s, "bab", "titogo", "TITO_S3")
357
358 if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC()); err != nil {
359 t.Fatal(err)
360 }
361 // Clobber the secret half with a plaintext env var of the same name.
362 if _, err := appconfig.Apply(ctx, s, "bab", "titogo",
363 map[string]appconfig.Var{"TITO_S3_SECRET_KEY": {Kind: "env", Value: "oops"}}, nil); err != nil {
364 t.Fatal(err)
365 }
366 missing, err := EnvDelivered(ctx, s, "bab", "titogo", "TITO_S3", false)
367 if err != nil || len(missing) != 1 || missing[0] != "TITO_S3_SECRET_KEY" {
368 t.Fatalf("wrong-kind secret: missing = %v, %v", missing, err)
369 }
370 // Same for a coordinate occupied by a sealed secret.
371 sealed, err := appconfig.Seal(pub, appconfig.Binding{Account: "bab", App: "titogo"}, []byte("carlos-store-tito-bab-titogo"))
372 if err != nil {
373 t.Fatal(err)
374 }
375 if _, err := appconfig.Apply(ctx, s, "bab", "titogo",
376 map[string]appconfig.Var{"TITO_S3_BUCKET": {Kind: "secret", Envelope: sealed}}, nil); err != nil {
377 t.Fatal(err)
378 }
379 if missing, err = EnvDelivered(ctx, s, "bab", "titogo", "TITO_S3", false); err != nil || len(missing) != 2 {
380 t.Fatalf("wrong-kind coordinate: missing = %v, %v", missing, err)
381 }
382
383 rec, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC())
384 if err != nil {
385 t.Fatal(err)
386 }
387 vars := mustVars(t, s, "bab", "titogo")
388 if v := vars["TITO_S3_BUCKET"]; v.Kind != "env" || v.Value != "carlos-store-tito-bab-titogo" {
389 t.Fatalf("bucket not repaired: %+v", v)
390 }
391 sk := vars["TITO_S3_SECRET_KEY"]
392 if sk.Kind != "secret" || sk.Value != "" {
393 t.Fatalf("secret not repaired: %+v", sk)
394 }
395 live := p.Users["carlos-store-bab-titogo"].Keys[rec.Provisioned.KeyID]
396 got, err := appconfig.Open(priv, appconfig.Binding{Account: "bab", App: "titogo"}, sk.Envelope)
397 if err != nil || string(got) != live {
398 t.Fatalf("repaired secret = %q, %v; want %q", got, err, live)
399 }
400 if missing, err = EnvDelivered(ctx, s, "bab", "titogo", "TITO_S3", false); err != nil || len(missing) != 0 {
401 t.Fatalf("after repair: missing = %v, %v", missing, err)
402 }
403}
404
405func TestValidBucketName(t *testing.T) {
406 for name, want := range map[string]bool{
407 "carlos-store-tito-bab-titogo": true,
408 "titogo-uploads": true,
409 "a.b-c9": true,
410 "abc": true,
411 "ab": false, // too short
412 "": false,
413 "UPPER-case": false,
414 "has_underscore": false,
415 "-leading": false,
416 "trailing-": false,
417 "has space": false,
418 "arn:aws:s3:::other/*": false,
419 "x" + string(make([]byte, 70)): false,
420 } {
421 if ValidBucketName(name) != want {
422 t.Errorf("ValidBucketName(%q) != %v", name, want)
423 }
424 }
425 // 63 legal, 64 not
426 if !ValidBucketName("a" + strings.Repeat("b", 61) + "c") {
427 t.Error("63-char name refused")
428 }
429 if ValidBucketName("a" + strings.Repeat("b", 62) + "c") {
430 t.Error("64-char name accepted")
431 }
432}
433
434// An adopt name is interpolated into the IAM policy ARN, so a bad one must
435// never reach the backend — nor be recorded as this app's bucket.
436func TestGrantRefusesInvalidBucketName(t *testing.T) {
437 ctx := context.Background()
438 s := release.NewDirStore(t.TempDir())
439 _, pub := testSealKeypair(t)
440 p := NewMemProvisioner("eu-west-1")
441 declare(t, s, "bab", "titogo", "TITO_S3")
442
443 _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "Other-Bucket/*", time.Now().UTC())
444 if err == nil || !strings.Contains(err.Error(), "invalid bucket name") {
445 t.Fatalf("adopt of an invalid name: %v", err)
446 }
447 if len(p.Buckets) != 0 || len(p.Users) != 0 {
448 t.Fatalf("refused grant touched the backend: %v %v", p.Buckets, p.Users)
449 }
450 rec, _, err := Get(ctx, s, "bab", "titogo")
451 if err != nil || rec == nil || rec.Grant != nil {
452 t.Fatalf("refused grant was recorded: %+v, %v", rec, err)
453 }
454 // A deployment whose prefix mints an illegal name is refused the same way.
455 d := testDeployment()
456 d.BucketPrefix = "Carlos_Store-"
457 if _, err := Grant(ctx, s, p, d, pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC()); err == nil ||
458 !strings.Contains(err.Error(), "invalid bucket name") {
459 t.Fatalf("minted invalid name: %v", err)
460 }
461}
462
463// The backend picks the secret's bytes. One that cannot be rendered into an
464// EnvironmentFile must be refused HERE, where the operator sees it, not hours
465// later on every host — and the refusal must not echo the secret.
466func TestGrantRefusesUnrenderableSecret(t *testing.T) {
467 ctx := context.Background()
468 s := release.NewDirStore(t.TempDir())
469 _, pub := testSealKeypair(t)
470 p := NewMemProvisioner("eu-west-1")
471 p.MintSecret = "bad\"secret\nwith\\newline"
472 declare(t, s, "bab", "titogo", "TITO_S3")
473
474 _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC())
475 if err == nil {
476 t.Fatal("unrenderable secret was delivered")
477 }
478 if !strings.Contains(err.Error(), "TITO_S3_SECRET_KEY") {
479 t.Fatalf("error does not name the variable: %v", err)
480 }
481 if strings.Contains(err.Error(), p.MintSecret) || strings.Contains(err.Error(), "bad") {
482 t.Fatalf("error echoed the secret: %v", err)
483 }
484 if rec, _, gerr := Get(ctx, s, "bab", "titogo"); gerr != nil || rec.Provisioned != nil {
485 t.Fatalf("record claims provisioned after a refused delivery: %+v, %v", rec, gerr)
486 }
487 if v := mustVars(t, s, "bab", "titogo"); len(v) != 0 {
488 t.Fatalf("config written despite refusal: %v", v)
489 }
490 // The wasted key is collected by the next walk's reap.
491 p.MintSecret = ""
492 rec, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC())
493 if err != nil {
494 t.Fatal(err)
495 }
496 if u := p.Users["carlos-store-bab-titogo"]; len(u.Keys) != 1 || u.Keys[rec.Provisioned.KeyID] == "" {
497 t.Fatalf("keys after recovery = %v (recorded %q)", u.Keys, rec.Provisioned.KeyID)
498 }
499}
500
501// Dropping a deployment's endpoint override must converge: the stale
502// _ENDPOINT is unset, not left pointing hosts at an address nobody uses.
503func TestGrantUnsetsStaleEndpoint(t *testing.T) {
504 ctx := context.Background()
505 s := release.NewDirStore(t.TempDir())
506 _, pub := testSealKeypair(t)
507 p := NewMemProvisioner("eu-west-1")
508 declare(t, s, "bab", "titogo", "TITO_S3")
509
510 withEndpoint := testDeployment()
511 withEndpoint.Endpoint = "https://s3.example.net"
512 if _, err := Grant(ctx, s, p, withEndpoint, pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC()); err != nil {
513 t.Fatal(err)
514 }
515 if _, ok := mustVars(t, s, "bab", "titogo")["TITO_S3_ENDPOINT"]; !ok {
516 t.Fatal("endpoint never delivered")
517 }
518 // Same app, deployment no longer has an endpoint: the record's endpoint no
519 // longer matches, so the walk must run rather than short-circuit.
520 rec, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC())
521 if err != nil {
522 t.Fatal(err)
523 }
524 if rec.Provisioned.Endpoint != "" {
525 t.Fatalf("record still carries the endpoint: %+v", rec.Provisioned)
526 }
527 if v, ok := mustVars(t, s, "bab", "titogo")["TITO_S3_ENDPOINT"]; ok {
528 t.Fatalf("stale endpoint left behind: %+v", v)
529 }
530 missing, err := EnvDelivered(ctx, s, "bab", "titogo", "TITO_S3", false)
531 if err != nil || len(missing) != 0 {
532 t.Fatalf("after endpoint drop: missing = %v, %v", missing, err)
533 }
534}
535
536func TestEnvDeliveredReportsMissing(t *testing.T) {
537 ctx := context.Background()
538 s := release.NewDirStore(t.TempDir())
539 _, pub := testSealKeypair(t)
540 p := NewMemProvisioner("eu-west-1")
541
542 missing, err := EnvDelivered(ctx, s, "bab", "titogo", "TITO_S3", false)
543 if err != nil {
544 t.Fatal(err)
545 }
546 if len(missing) != 4 {
547 t.Fatalf("undelivered app: missing = %v, want the 4 non-endpoint names", missing)
548 }
549
550 declare(t, s, "bab", "titogo", "TITO_S3")
551 if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC()); err != nil {
552 t.Fatal(err)
553 }
554 if missing, err = EnvDelivered(ctx, s, "bab", "titogo", "TITO_S3", false); err != nil || len(missing) != 0 {
555 t.Fatalf("after grant: missing = %v, %v", missing, err)
556 }
557 // endpoint wanted but never delivered (deployment has no endpoint)
558 if missing, err = EnvDelivered(ctx, s, "bab", "titogo", "TITO_S3", true); err != nil ||
559 len(missing) != 1 || missing[0] != "TITO_S3_ENDPOINT" {
560 t.Fatalf("endpoint wanted: missing = %v, %v", missing, err)
561 }
562 if _, err := appconfig.Apply(ctx, s, "bab", "titogo", nil, []string{"TITO_S3_SECRET_KEY"}); err != nil {
563 t.Fatal(err)
564 }
565 if missing, err = EnvDelivered(ctx, s, "bab", "titogo", "TITO_S3", false); err != nil ||
566 len(missing) != 1 || missing[0] != "TITO_S3_SECRET_KEY" {
567 t.Fatalf("after unset: missing = %v, %v", missing, err)
568 }
569}
570
571// rotateFixture grants a store and returns the provisioned record, so each
572// rotation test starts from the one state Rotate is defined against.
573func rotateFixture(t *testing.T, s release.Store, pub *ecdh.PublicKey, p *MemProvisioner) *Record {
574 t.Helper()
575 declare(t, s, "bab", "titogo", "TITO_S3")
576 rec, err := Grant(context.Background(), s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC())
577 if err != nil {
578 t.Fatal(err)
579 }
580 return rec
581}
582
583// openVar decrypts a delivered sealed var, so a test asserts on what the app
584// would actually read rather than on the record alone.
585func openVar(t *testing.T, priv *ecdh.PrivateKey, s release.Store, acct, app, name string) string {
586 t.Helper()
587 v := mustVars(t, s, acct, app)[name]
588 if v.Kind != "secret" || v.Envelope == "" {
589 t.Fatalf("%s = %+v, want a sealed secret", name, v)
590 }
591 plain, err := appconfig.Open(priv, appconfig.Binding{Account: acct, App: app}, v.Envelope)
592 if err != nil {
593 t.Fatalf("open %s: %v", name, err)
594 }
595 return string(plain)
596}
597
598// The whole point of the two-key window: the app keeps running on the key it
599// already holds (in-flight requests, un-restarted processes) while the new one
600// is delivered. The old key stays ACTIVE until an explicit finish.
601func TestRotateKeepsOldKeyActive(t *testing.T) {
602 ctx := context.Background()
603 s := release.NewDirStore(t.TempDir())
604 priv, pub := testSealKeypair(t)
605 p := NewMemProvisioner("eu-west-1")
606 granted := rotateFixture(t, s, pub, p)
607 oldID := granted.Provisioned.KeyID
608
609 now := time.Now().UTC().Add(time.Hour)
610 rec, err := Rotate(ctx, s, p, pub, "bab", "titogo", now)
611 if err != nil {
612 t.Fatal(err)
613 }
614 if rec.Provisioned.KeyID == oldID {
615 t.Fatalf("rotate did not mint a new key: still %q", rec.Provisioned.KeyID)
616 }
617 if rec.Provisioned.PrevKeyID != oldID {
618 t.Fatalf("PrevKeyID = %q, want the old key %q", rec.Provisioned.PrevKeyID, oldID)
619 }
620 if !rec.Provisioned.At.Equal(now) {
621 t.Fatalf("At = %v, want %v", rec.Provisioned.At, now)
622 }
623 // coordinates are carried, never re-derived
624 if rec.Provisioned.Bucket != granted.Provisioned.Bucket ||
625 rec.Provisioned.Region != granted.Provisioned.Region ||
626 rec.Provisioned.IAMUser != granted.Provisioned.IAMUser ||
627 rec.Provisioned.UserARN != granted.Provisioned.UserARN {
628 t.Fatalf("rotate moved the store: %+v", rec.Provisioned)
629 }
630
631 u := p.Users["carlos-store-bab-titogo"]
632 if len(u.Keys) != 2 {
633 t.Fatalf("keys after rotate = %v, want both old and new", u.Keys)
634 }
635 if _, ok := u.Keys[oldID]; !ok {
636 t.Fatalf("old key %q was deleted by rotate: %v", oldID, u.Keys)
637 }
638 if _, ok := u.Keys[rec.Provisioned.KeyID]; !ok {
639 t.Fatalf("new key %q is not live: %v", rec.Provisioned.KeyID, u.Keys)
640 }
641
642 // the app reads the NEW pair
643 if got := openVar(t, priv, s, "bab", "titogo", "TITO_S3_ACCESS_KEY"); got != rec.Provisioned.KeyID {
644 t.Fatalf("delivered access key = %q, want %q", got, rec.Provisioned.KeyID)
645 }
646 if got := openVar(t, priv, s, "bab", "titogo", "TITO_S3_SECRET_KEY"); got != u.Keys[rec.Provisioned.KeyID] {
647 t.Fatalf("delivered secret = %q, want %q", got, u.Keys[rec.Provisioned.KeyID])
648 }
649 // coordinates still delivered alongside, in the same config
650 if v := mustVars(t, s, "bab", "titogo")["TITO_S3_BUCKET"]; v.Kind != "env" || v.Value != granted.Provisioned.Bucket {
651 t.Fatalf("TITO_S3_BUCKET = %+v", v)
652 }
653
654 stored, _, err := Get(ctx, s, "bab", "titogo")
655 if err != nil || stored == nil || stored.Provisioned == nil ||
656 stored.Provisioned.KeyID != rec.Provisioned.KeyID || stored.Provisioned.PrevKeyID != oldID {
657 t.Fatalf("stored record = %+v, %v", stored, err)
658 }
659}
660
661// AWS allows two access keys per user, full stop. A member who rotates twice
662// without finishing must not wedge at the ceiling: the second rotate finishes
663// the pending one first, so the estate is never more than two keys.
664func TestRotateTwiceReapsPending(t *testing.T) {
665 ctx := context.Background()
666 s := release.NewDirStore(t.TempDir())
667 priv, pub := testSealKeypair(t)
668 p := NewMemProvisioner("eu-west-1")
669 first := rotateFixture(t, s, pub, p).Provisioned.KeyID
670 user := "carlos-store-bab-titogo"
671
672 second, err := Rotate(ctx, s, p, pub, "bab", "titogo", time.Now().UTC())
673 if err != nil {
674 t.Fatal(err)
675 }
676 if len(p.Users[user].Keys) != 2 {
677 t.Fatalf("keys after first rotate = %v", p.Users[user].Keys)
678 }
679
680 third, err := Rotate(ctx, s, p, pub, "bab", "titogo", time.Now().UTC())
681 if err != nil {
682 t.Fatal(err)
683 }
684 keys := p.Users[user].Keys
685 if len(keys) != 2 {
686 t.Fatalf("keys after second rotate = %v, want exactly 2", keys)
687 }
688 if _, ok := keys[first]; ok {
689 t.Fatalf("the finished key %q is still live: %v", first, keys)
690 }
691 if _, ok := keys[second.Provisioned.KeyID]; !ok {
692 t.Fatalf("pending key %q was deleted: %v", second.Provisioned.KeyID, keys)
693 }
694 if _, ok := keys[third.Provisioned.KeyID]; !ok {
695 t.Fatalf("new key %q is not live: %v", third.Provisioned.KeyID, keys)
696 }
697 if third.Provisioned.PrevKeyID != second.Provisioned.KeyID {
698 t.Fatalf("PrevKeyID = %q, want %q", third.Provisioned.PrevKeyID, second.Provisioned.KeyID)
699 }
700 if got := openVar(t, priv, s, "bab", "titogo", "TITO_S3_ACCESS_KEY"); got != third.Provisioned.KeyID {
701 t.Fatalf("delivered access key = %q, want %q", got, third.Provisioned.KeyID)
702 }
703}
704
705// A rotate that mints and then fails to reseal leaves a key the record does
706// not name. Re-running must collect it (the reap) rather than mint into a full
707// estate, and the app must end up holding the key the record names.
708func TestRotateResumesAfterResealFailure(t *testing.T) {
709 ctx := context.Background()
710 priv, pub := testSealKeypair(t)
711 s := &failingStore{Store: release.NewDirStore(t.TempDir())}
712 p := NewMemProvisioner("eu-west-1")
713 granted := rotateFixture(t, s, pub, p)
714 oldID := granted.Provisioned.KeyID
715 user := "carlos-store-bab-titogo"
716
717 s.failKeyOnce = appconfig.Key("bab", "titogo")
718 if _, err := Rotate(ctx, s, p, pub, "bab", "titogo", time.Now().UTC()); err == nil {
719 t.Fatal("reseal failure: Rotate returned no error")
720 }
721 // The record must still name the key the app is holding — the new one was
722 // never delivered, so claiming it would strand the app on a dead name.
723 half, _, err := Get(ctx, s, "bab", "titogo")
724 if err != nil || half == nil || half.Provisioned == nil ||
725 half.Provisioned.KeyID != oldID || half.Provisioned.PrevKeyID != "" {
726 t.Fatalf("record after failed reseal = %+v, %v", half, err)
727 }
728 if len(p.Users[user].Keys) > 2 {
729 t.Fatalf("ceiling breached mid-failure: %v", p.Users[user].Keys)
730 }
731 if got := openVar(t, priv, s, "bab", "titogo", "TITO_S3_ACCESS_KEY"); got != oldID {
732 t.Fatalf("failed rotate moved the app's key to %q, want %q", got, oldID)
733 }
734
735 rec, err := Rotate(ctx, s, p, pub, "bab", "titogo", time.Now().UTC())
736 if err != nil {
737 t.Fatalf("resume after reseal failure: %v", err)
738 }
739 keys := p.Users[user].Keys
740 if len(keys) != 2 {
741 t.Fatalf("keys after resume = %v, want exactly 2 (stray reaped)", keys)
742 }
743 if rec.Provisioned.PrevKeyID != oldID {
744 t.Fatalf("PrevKeyID = %q, want %q", rec.Provisioned.PrevKeyID, oldID)
745 }
746 if _, ok := keys[rec.Provisioned.KeyID]; !ok {
747 t.Fatalf("recorded key %q is not live: %v", rec.Provisioned.KeyID, keys)
748 }
749 if got := openVar(t, priv, s, "bab", "titogo", "TITO_S3_ACCESS_KEY"); got != rec.Provisioned.KeyID {
750 t.Fatalf("delivered access key = %q, want %q", got, rec.Provisioned.KeyID)
751 }
752 if got := openVar(t, priv, s, "bab", "titogo", "TITO_S3_SECRET_KEY"); got != keys[rec.Provisioned.KeyID] {
753 t.Fatalf("delivered secret = %q, want %q", got, keys[rec.Provisioned.KeyID])
754 }
755}
756
757// Finish is the member saying "nothing is using the old key any more". It
758// deletes exactly that key and nothing else, delivers nothing, and is not
759// repeatable — a second finish has no key to name and must say so.
760func TestRotateFinishDeletesPrev(t *testing.T) {
761 ctx := context.Background()
762 s := release.NewDirStore(t.TempDir())
763 priv, pub := testSealKeypair(t)
764 p := NewMemProvisioner("eu-west-1")
765 oldID := rotateFixture(t, s, pub, p).Provisioned.KeyID
766 user := "carlos-store-bab-titogo"
767
768 rotated, err := Rotate(ctx, s, p, pub, "bab", "titogo", time.Now().UTC())
769 if err != nil {
770 t.Fatal(err)
771 }
772 before := mustVars(t, s, "bab", "titogo")
773
774 rec, err := RotateFinish(ctx, s, p, "bab", "titogo")
775 if err != nil {
776 t.Fatal(err)
777 }
778 if rec.Provisioned.PrevKeyID != "" {
779 t.Fatalf("PrevKeyID after finish = %q", rec.Provisioned.PrevKeyID)
780 }
781 if rec.Provisioned.KeyID != rotated.Provisioned.KeyID {
782 t.Fatalf("finish changed the live key: %q", rec.Provisioned.KeyID)
783 }
784 keys := p.Users[user].Keys
785 if len(keys) != 1 {
786 t.Fatalf("keys after finish = %v, want just the live one", keys)
787 }
788 if _, ok := keys[oldID]; ok {
789 t.Fatalf("finished key %q still live: %v", oldID, keys)
790 }
791 if _, ok := keys[rotated.Provisioned.KeyID]; !ok {
792 t.Fatalf("live key %q deleted by finish: %v", rotated.Provisioned.KeyID, keys)
793 }
794 // Finish delivers nothing: the app's config is untouched.
795 after := mustVars(t, s, "bab", "titogo")
796 if len(before) != len(after) {
797 t.Fatalf("finish rewrote the var set: %v then %v", before, after)
798 }
799 for k, v := range before {
800 if after[k] != v {
801 t.Fatalf("finish rewrote %s: %+v then %+v", k, v, after[k])
802 }
803 }
804 if got := openVar(t, priv, s, "bab", "titogo", "TITO_S3_ACCESS_KEY"); got != rec.Provisioned.KeyID {
805 t.Fatalf("delivered access key = %q, want %q", got, rec.Provisioned.KeyID)
806 }
807
808 if _, err := RotateFinish(ctx, s, p, "bab", "titogo"); !errors.Is(err, ErrNoPendingFinish) {
809 t.Fatalf("second finish: %v, want ErrNoPendingFinish", err)
810 }
811 if len(p.Users[user].Keys) != 1 {
812 t.Fatalf("refused finish touched the backend: %v", p.Users[user].Keys)
813 }
814
815 stored, _, err := Get(ctx, s, "bab", "titogo")
816 if err != nil || stored == nil || stored.Provisioned == nil || stored.Provisioned.PrevKeyID != "" {
817 t.Fatalf("stored record = %+v, %v", stored, err)
818 }
819}
820
821// Neither walk invents a store: there is nothing to rotate until a grant has
822// provisioned one, and both must refuse without touching the backend.
823func TestRotateUnprovisionedRefused(t *testing.T) {
824 ctx := context.Background()
825 s := release.NewDirStore(t.TempDir())
826 _, pub := testSealKeypair(t)
827 p := NewMemProvisioner("eu-west-1")
828 declare(t, s, "bab", "titogo", "TITO_S3")
829
830 if _, err := Rotate(ctx, s, p, pub, "bab", "titogo", time.Now().UTC()); !errors.Is(err, ErrNotProvisioned) {
831 t.Fatalf("Rotate on a declared-only store: %v, want ErrNotProvisioned", err)
832 }
833 if _, err := RotateFinish(ctx, s, p, "bab", "titogo"); !errors.Is(err, ErrNotProvisioned) {
834 t.Fatalf("RotateFinish on a declared-only store: %v, want ErrNotProvisioned", err)
835 }
836 // and on an app with no record at all
837 if _, err := Rotate(ctx, s, p, pub, "bab", "nostore", time.Now().UTC()); !errors.Is(err, ErrNotProvisioned) {
838 t.Fatalf("Rotate on a store-less app: %v, want ErrNotProvisioned", err)
839 }
840 if _, err := RotateFinish(ctx, s, p, "bab", "nostore"); !errors.Is(err, ErrNotProvisioned) {
841 t.Fatalf("RotateFinish on a store-less app: %v, want ErrNotProvisioned", err)
842 }
843 if len(p.Buckets) != 0 || len(p.Users) != 0 {
844 t.Fatalf("refused rotation touched the backend: %v %v", p.Buckets, p.Users)
845 }
846 if v := mustVars(t, s, "bab", "titogo"); len(v) != 0 {
847 t.Fatalf("refused rotation wrote config: %v", v)
848 }
849}
850
851// A rotation changes credentials and NOTHING else. The coordinates it
852// redelivers come from the record, so an endpoint-bearing store must come out
853// of a rotate still pointing at its endpoint — a rotate that re-derived them
854// (or dropped them) would silently move every host onto the backend's own
855// address, which is a data-plane outage with a green rotate next to it.
856func TestRotateKeepsRecordedCoordinates(t *testing.T) {
857 ctx := context.Background()
858 s := release.NewDirStore(t.TempDir())
859 priv, pub := testSealKeypair(t)
860 p := NewMemProvisioner("eu-central-1")
861 declare(t, s, "bab", "titogo", "TITO_S3")
862
863 d := testDeployment()
864 d.Endpoint = "https://s3.example.net"
865 granted, err := Grant(ctx, s, p, d, pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC())
866 if err != nil {
867 t.Fatal(err)
868 }
869 if granted.Provisioned.Endpoint != d.Endpoint || granted.Provisioned.Region != "eu-central-1" {
870 t.Fatalf("fixture did not record the coordinates: %+v", granted.Provisioned)
871 }
872
873 // Rotate takes no Deployment: the only source for these values is the
874 // record, so this is the assertion that it reads them from there.
875 rec, err := Rotate(ctx, s, p, pub, "bab", "titogo", time.Now().UTC())
876 if err != nil {
877 t.Fatal(err)
878 }
879 if rec.Provisioned.Endpoint != d.Endpoint || rec.Provisioned.Region != "eu-central-1" ||
880 rec.Provisioned.Bucket != granted.Provisioned.Bucket {
881 t.Fatalf("rotate moved the coordinates: %+v", rec.Provisioned)
882 }
883 vars := mustVars(t, s, "bab", "titogo")
884 if v := vars["TITO_S3_ENDPOINT"]; v.Kind != "env" || v.Value != d.Endpoint {
885 t.Fatalf("TITO_S3_ENDPOINT after rotate = %+v, want %q", v, d.Endpoint)
886 }
887 if v := vars["TITO_S3_REGION"]; v.Kind != "env" || v.Value != "eu-central-1" {
888 t.Fatalf("TITO_S3_REGION after rotate = %+v", v)
889 }
890 if v := vars["TITO_S3_BUCKET"]; v.Kind != "env" || v.Value != granted.Provisioned.Bucket {
891 t.Fatalf("TITO_S3_BUCKET after rotate = %+v", v)
892 }
893 if got := openVar(t, priv, s, "bab", "titogo", "TITO_S3_ACCESS_KEY"); got != rec.Provisioned.KeyID {
894 t.Fatalf("delivered access key = %q, want the new id %q", got, rec.Provisioned.KeyID)
895 }
896 if missing, err := EnvDelivered(ctx, s, "bab", "titogo", "TITO_S3", true); err != nil || len(missing) != 0 {
897 t.Fatalf("after rotate: missing = %v, %v", missing, err)
898 }
899}
900
901// racingStore lands a competing write on the record immediately BEFORE the
902// walk's own Put reaches the store — the interleaving that turns the walk's
903// etag stale. One shot, so only the write under test races.
904type racingStore struct {
905 release.Store
906 raceKeyOnce string
907 race func(inner release.Store)
908}
909
910func (r *racingStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) {
911 if key != "" && key == r.raceKeyOnce {
912 r.raceKeyOnce = ""
913 r.race(r.Store)
914 }
915 return r.Store.Put(ctx, key, body, ifMatch)
916}
917
918// The window between a rotate's Get and its Put is real: the console's
919// declare/grant doors write the same object. Losing that CAS outright would
920// leave the record naming the OLD key while the app runs on the new one —
921// and the next rotate's reap would then delete the key actually in use. So
922// the walk retries against a re-read record, and the competitor's change to
923// the blocks it does not own must survive.
924func TestRotateRetriesLostCAS(t *testing.T) {
925 ctx := context.Background()
926 priv, pub := testSealKeypair(t)
927 inner := release.NewDirStore(t.TempDir())
928 s := &racingStore{Store: inner}
929 p := NewMemProvisioner("eu-west-1")
930 oldID := rotateFixture(t, s, pub, p).Provisioned.KeyID
931
932 // A competing writer touches a block this walk does not own.
933 s.raceKeyOnce = Key("bab", "titogo")
934 s.race = func(in release.Store) {
935 other, etag, err := Get(ctx, in, "bab", "titogo")
936 if err != nil {
937 t.Fatal(err)
938 }
939 other.Declared.By = "someone-else@example.com"
940 if _, err := Put(ctx, in, "bab", "titogo", other, etag); err != nil {
941 t.Fatal(err)
942 }
943 }
944
945 rec, err := Rotate(ctx, s, p, pub, "bab", "titogo", time.Now().UTC())
946 if err != nil {
947 t.Fatalf("rotate lost its CAS and gave up: %v", err)
948 }
949 if rec.Provisioned.PrevKeyID != oldID || rec.Provisioned.KeyID == oldID {
950 t.Fatalf("provisioned = %+v", rec.Provisioned)
951 }
952 if rec.Declared.By != "someone-else@example.com" {
953 t.Fatalf("retry clobbered the competing write: Declared.By = %q", rec.Declared.By)
954 }
955 stored, _, err := Get(ctx, s, "bab", "titogo")
956 if err != nil || stored.Declared.By != "someone-else@example.com" ||
957 stored.Provisioned.KeyID != rec.Provisioned.KeyID || stored.Provisioned.PrevKeyID != oldID {
958 t.Fatalf("stored record = %+v / %+v, %v", stored.Declared, stored.Provisioned, err)
959 }
960 // The record and the app agree on the live key — the whole point.
961 if got := openVar(t, priv, s, "bab", "titogo", "TITO_S3_ACCESS_KEY"); got != stored.Provisioned.KeyID {
962 t.Fatalf("delivered access key = %q, record says %q", got, stored.Provisioned.KeyID)
963 }
964}
965
966// Rotate-vs-rotate is the case the retry must NOT paper over: the competitor
967// delivered a credential of its own, so overwriting its record would strand
968// that key exactly as a lost CAS would strand ours. The conflict is surfaced.
969func TestRotateRefusesRetryAgainstAnotherRotation(t *testing.T) {
970 ctx := context.Background()
971 _, pub := testSealKeypair(t)
972 inner := release.NewDirStore(t.TempDir())
973 s := &racingStore{Store: inner}
974 p := NewMemProvisioner("eu-west-1")
975 oldID := rotateFixture(t, s, pub, p).Provisioned.KeyID
976
977 s.raceKeyOnce = Key("bab", "titogo")
978 s.race = func(in release.Store) {
979 other, etag, err := Get(ctx, in, "bab", "titogo")
980 if err != nil {
981 t.Fatal(err)
982 }
983 other.Provisioned.KeyID = "MEMKEY99" // another rotation landed first
984 other.Provisioned.PrevKeyID = oldID
985 if _, err := Put(ctx, in, "bab", "titogo", other, etag); err != nil {
986 t.Fatal(err)
987 }
988 }
989
990 if _, err := Rotate(ctx, s, p, pub, "bab", "titogo", time.Now().UTC()); !errors.Is(err, release.ErrCASConflict) {
991 t.Fatalf("rotate over a concurrent rotation: %v, want ErrCASConflict", err)
992 }
993 stored, _, err := Get(ctx, s, "bab", "titogo")
994 if err != nil || stored.Provisioned.KeyID != "MEMKEY99" || stored.Provisioned.PrevKeyID != oldID {
995 t.Fatalf("the competing rotation was clobbered: %+v, %v", stored.Provisioned, err)
996 }
997}
998
999// teardownFixture grants a fresh store and returns its record — the estate a
1000// teardown has to dismantle.
1001func teardownFixture(t *testing.T, s release.Store, pub *ecdh.PublicKey, p *MemProvisioner) *Record {
1002 t.Helper()
1003 declare(t, s, "bab", "titogo", "TITO_S3")
1004 rec, err := Grant(context.Background(), s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC())
1005 if err != nil {
1006 t.Fatal(err)
1007 }
1008 return rec
1009}
1010
1011func TestTeardownFullSweep(t *testing.T) {
1012 ctx := context.Background()
1013 s := release.NewDirStore(t.TempDir())
1014 _, pub := testSealKeypair(t)
1015 p := NewMemProvisioner("eu-west-1")
1016 rec := teardownFixture(t, s, pub, p)
1017 // A bucket with data in it is the case that matters: DeleteBucket alone
1018 // would fail on a real backend, so the walk must empty it first.
1019 p.Buckets[rec.Provisioned.Bucket]["uploads/one.txt"] = []byte("payload")
1020
1021 if err := Teardown(ctx, s, p, nil, "bab", "titogo"); err != nil {
1022 t.Fatal(err)
1023 }
1024 if len(p.Buckets) != 0 {
1025 t.Fatalf("buckets after teardown = %v", p.Buckets)
1026 }
1027 if len(p.Users) != 0 {
1028 t.Fatalf("users after teardown = %v", p.Users)
1029 }
1030 for name := range mustVars(t, s, "bab", "titogo") {
1031 if strings.HasPrefix(name, "TITO_S3_") {
1032 t.Fatalf("delivered var %q survived teardown", name)
1033 }
1034 }
1035 stored, _, err := Get(ctx, s, "bab", "titogo")
1036 if err != nil || stored != nil {
1037 t.Fatalf("record after teardown = %+v, %v", stored, err)
1038 }
1039}
1040
1041func TestTeardownResumable(t *testing.T) {
1042 for _, method := range []string{"EmptyBucket", "DeleteBucket", "DeleteUser"} {
1043 t.Run(method, func(t *testing.T) {
1044 ctx := context.Background()
1045 s := release.NewDirStore(t.TempDir())
1046 _, pub := testSealKeypair(t)
1047 p := NewMemProvisioner("eu-west-1")
1048 rec := teardownFixture(t, s, pub, p)
1049 p.Buckets[rec.Provisioned.Bucket]["uploads/one.txt"] = []byte("payload")
1050
1051 p.FailNext[method] = errors.New("injected: " + method + " failed")
1052 if err := Teardown(ctx, s, p, nil, "bab", "titogo"); err == nil {
1053 t.Fatalf("%s failure: Teardown returned no error", method)
1054 }
1055 // The record is the ONLY name for what is left standing (§9), so a
1056 // half-failed teardown must leave it intact and still provisioned.
1057 half, _, err := Get(ctx, s, "bab", "titogo")
1058 if err != nil || half == nil || half.Provisioned == nil {
1059 t.Fatalf("record after %s failure = %+v, %v", method, half, err)
1060 }
1061 if half.Provisioned.Bucket != rec.Provisioned.Bucket || half.Provisioned.IAMUser != rec.Provisioned.IAMUser {
1062 t.Fatalf("record coordinates changed after %s failure: %+v", method, half.Provisioned)
1063 }
1064
1065 if err := Teardown(ctx, s, p, nil, "bab", "titogo"); err != nil {
1066 t.Fatalf("resume after %s failure: %v", method, err)
1067 }
1068 if len(p.Buckets) != 0 || len(p.Users) != 0 {
1069 t.Fatalf("estate after resume: buckets %v users %v", p.Buckets, p.Users)
1070 }
1071 for name := range mustVars(t, s, "bab", "titogo") {
1072 if strings.HasPrefix(name, "TITO_S3_") {
1073 t.Fatalf("delivered var %q survived teardown", name)
1074 }
1075 }
1076 stored, _, err := Get(ctx, s, "bab", "titogo")
1077 if err != nil || stored != nil {
1078 t.Fatalf("record after resume = %+v, %v", stored, err)
1079 }
1080 })
1081 }
1082}
1083
1084func TestTeardownDeclaredOnly(t *testing.T) {
1085 ctx := context.Background()
1086 s := release.NewDirStore(t.TempDir())
1087 p := NewMemProvisioner("eu-west-1")
1088 declare(t, s, "bab", "titogo", "TITO_S3")
1089
1090 if err := Teardown(ctx, s, p, nil, "bab", "titogo"); err != nil {
1091 t.Fatal(err)
1092 }
1093 stored, _, err := Get(ctx, s, "bab", "titogo")
1094 if err != nil || stored != nil {
1095 t.Fatalf("declared-only record after teardown = %+v, %v", stored, err)
1096 }
1097 if len(p.Buckets) != 0 || len(p.Users) != 0 {
1098 t.Fatalf("declared-only teardown touched the backend: %v %v", p.Buckets, p.Users)
1099 }
1100}
1101
1102// A deployment with no backend may still tear down a DECLARATION — there is
1103// nothing outside the bucket to act on — but an estate refuses instead of
1104// dereferencing the nil backend. The console's destroy door decides from a
1105// record it read a round trip earlier, so a grant landing in between arrives
1106// here as exactly this shape, and a panic would be the difference between a
1107// refusal and a 500 over a live bucket.
1108func TestTeardownWithNoProvisioner(t *testing.T) {
1109 ctx := context.Background()
1110 s := release.NewDirStore(t.TempDir())
1111 declare(t, s, "bab", "titogo", "TITO_S3")
1112
1113 if err := Teardown(ctx, s, nil, nil, "bab", "titogo"); err != nil {
1114 t.Fatalf("declared-only teardown with no provisioner: %v", err)
1115 }
1116
1117 // Now the same walk over a grant that DID create backend estate.
1118 declare(t, s, "bab", "titogo", "TITO_S3")
1119 rec, etag, err := Get(ctx, s, "bab", "titogo")
1120 if err != nil {
1121 t.Fatal(err)
1122 }
1123 rec.Grant = &GrantBlock{Backend: "mem", Bucket: "carlos-bab-titogo", At: time.Now(), By: "op@keymail.dev"}
1124 if _, err := Put(ctx, s, "bab", "titogo", rec, etag); err != nil {
1125 t.Fatal(err)
1126 }
1127 if err := Teardown(ctx, s, nil, nil, "bab", "titogo"); err == nil {
1128 t.Fatal("teardown of a granted store with no provisioner returned nil — it cannot have destroyed anything")
1129 }
1130 if stored, _, err := Get(ctx, s, "bab", "titogo"); err != nil || stored == nil || stored.Grant == nil {
1131 t.Fatalf("the refused teardown did not leave the record intact: %+v %v", stored, err)
1132 }
1133}
1134
1135func TestTeardownIdempotentOnAbsent(t *testing.T) {
1136 ctx := context.Background()
1137 s := release.NewDirStore(t.TempDir())
1138 p := NewMemProvisioner("eu-west-1")
1139 if err := Teardown(ctx, s, p, nil, "bab", "titogo"); err != nil {
1140 t.Fatalf("teardown with no record: %v", err)
1141 }
1142 if len(p.Buckets) != 0 || len(p.Users) != 0 {
1143 t.Fatalf("teardown with no record touched the backend: %v %v", p.Buckets, p.Users)
1144 }
1145}
1146
1147func TestTeardownConfigObjectAbsent(t *testing.T) {
1148 ctx := context.Background()
1149 s := release.NewDirStore(t.TempDir())
1150 p := NewMemProvisioner("eu-west-1")
1151 // A provisioned record whose app was never configured — hand-crafted, so
1152 // no config.json exists at all. appconfig REFUSES an unset-only apply on a
1153 // nonexistent object, and that refusal must not strand the estate.
1154 if _, err := p.EnsureBucket(ctx, "carlos-store-tito-bab-titogo"); err != nil {
1155 t.Fatal(err)
1156 }
1157 arn, err := p.EnsureUser(ctx, "carlos-store-bab-titogo", UserPath, "arn:aws:s3:::carlos-store-tito-bab-titogo", nil)
1158 if err != nil {
1159 t.Fatal(err)
1160 }
1161 rec := &Record{
1162 Declared: &Declared{EnvPrefix: "TITO_S3", At: time.Now().UTC(), By: "member@example.com"},
1163 Grant: &GrantBlock{Backend: "aws", Bucket: "carlos-store-tito-bab-titogo", At: time.Now().UTC(), By: "op"},
1164 Provisioned: &Provisioned{Bucket: "carlos-store-tito-bab-titogo", Region: "eu-west-1",
1165 IAMUser: "carlos-store-bab-titogo", UserARN: arn, KeyID: "MEMKEY1", At: time.Now().UTC()},
1166 }
1167 if _, err := Put(ctx, s, "bab", "titogo", rec, ""); err != nil {
1168 t.Fatal(err)
1169 }
1170
1171 if err := Teardown(ctx, s, p, nil, "bab", "titogo"); err != nil {
1172 t.Fatalf("teardown with no config object: %v", err)
1173 }
1174 if len(p.Buckets) != 0 || len(p.Users) != 0 {
1175 t.Fatalf("estate after teardown: buckets %v users %v", p.Buckets, p.Users)
1176 }
1177 stored, _, err := Get(ctx, s, "bab", "titogo")
1178 if err != nil || stored != nil {
1179 t.Fatalf("record after teardown = %+v, %v", stored, err)
1180 }
1181 // And no phantom config object was minted on the way through.
1182 if _, _, err := s.Get(ctx, appconfig.Key("bab", "titogo")); !errors.Is(err, os.ErrNotExist) {
1183 t.Fatalf("teardown minted a config object: %v", err)
1184 }
1185}
1186
1187// crashedGrant reproduces a grant that created and delivered everything and
1188// then lost its step-6 record Put: bucket, user, key and delivered vars all
1189// live, record carrying Declared+Grant and NO provisioned block.
1190func crashedGrant(t *testing.T, s release.Store, p *MemProvisioner) (bucket, user string) {
1191 t.Helper()
1192 ctx := context.Background()
1193 bucket, user = "carlos-store-tito-bab-titogo", UserName("bab", "titogo")
1194 if _, err := p.EnsureBucket(ctx, bucket); err != nil {
1195 t.Fatal(err)
1196 }
1197 if _, err := p.EnsureUser(ctx, user, UserPath, "arn:aws:s3:::"+bucket, nil); err != nil {
1198 t.Fatal(err)
1199 }
1200 id, _, err := p.MintKey(ctx, user)
1201 if err != nil {
1202 t.Fatal(err)
1203 }
1204 // writeEnv had already landed when the record write lost — the credentials
1205 // are in config.json under names no record now explains.
1206 if _, err := appconfig.Apply(ctx, s, "bab", "titogo", map[string]appconfig.Var{
1207 "TITO_S3_BUCKET": {Kind: "env", Value: bucket},
1208 "TITO_S3_REGION": {Kind: "env", Value: "eu-west-1"},
1209 "TITO_S3_ACCESS_KEY": {Kind: "env", Value: id},
1210 }, nil); err != nil {
1211 t.Fatal(err)
1212 }
1213 rec := &Record{
1214 Declared: &Declared{EnvPrefix: "TITO_S3", At: time.Now().UTC(), By: "member@example.com"},
1215 Grant: &GrantBlock{Backend: "aws", Bucket: bucket, At: time.Now().UTC(), By: "op"},
1216 }
1217 if _, err := Put(ctx, s, "bab", "titogo", rec, ""); err != nil {
1218 t.Fatal(err)
1219 }
1220 return bucket, user
1221}
1222
1223// A grant block with no provisioned block is NOT "nothing to tear down" — it
1224// is the shape a crash at Grant's final record write leaves over a live
1225// estate. Deleting the record alone would strand a billing bucket and a live
1226// credential nothing names.
1227func TestTeardownCrashedGrantSweep(t *testing.T) {
1228 ctx := context.Background()
1229 s := release.NewDirStore(t.TempDir())
1230 p := NewMemProvisioner("eu-west-1")
1231 bucket, _ := crashedGrant(t, s, p)
1232 p.Buckets[bucket]["uploads/one.txt"] = []byte("payload")
1233
1234 if err := Teardown(ctx, s, p, nil, "bab", "titogo"); err != nil {
1235 t.Fatal(err)
1236 }
1237 if len(p.Buckets) != 0 || len(p.Users) != 0 {
1238 t.Fatalf("crashed grant left standing: buckets %v users %v", p.Buckets, p.Users)
1239 }
1240 for name := range mustVars(t, s, "bab", "titogo") {
1241 if strings.HasPrefix(name, "TITO_S3_") {
1242 t.Fatalf("delivered var %q survived teardown", name)
1243 }
1244 }
1245 stored, _, err := Get(ctx, s, "bab", "titogo")
1246 if err != nil || stored != nil {
1247 t.Fatalf("record after teardown = %+v, %v", stored, err)
1248 }
1249}
1250
1251func TestTeardownCrashedGrantResumable(t *testing.T) {
1252 ctx := context.Background()
1253 s := release.NewDirStore(t.TempDir())
1254 p := NewMemProvisioner("eu-west-1")
1255 bucket, _ := crashedGrant(t, s, p)
1256 p.Buckets[bucket]["uploads/one.txt"] = []byte("payload")
1257
1258 p.FailNext["DeleteBucket"] = errors.New("injected: DeleteBucket failed")
1259 if err := Teardown(ctx, s, p, nil, "bab", "titogo"); err == nil {
1260 t.Fatal("DeleteBucket failure: Teardown returned no error")
1261 }
1262 half, _, err := Get(ctx, s, "bab", "titogo")
1263 if err != nil || half == nil || half.Grant == nil || half.Grant.Bucket != bucket {
1264 t.Fatalf("record after DeleteBucket failure = %+v, %v", half, err)
1265 }
1266
1267 if err := Teardown(ctx, s, p, nil, "bab", "titogo"); err != nil {
1268 t.Fatalf("resume after DeleteBucket failure: %v", err)
1269 }
1270 if len(p.Buckets) != 0 || len(p.Users) != 0 {
1271 t.Fatalf("estate after resume: buckets %v users %v", p.Buckets, p.Users)
1272 }
1273 stored, _, err := Get(ctx, s, "bab", "titogo")
1274 if err != nil || stored != nil {
1275 t.Fatalf("record after resume = %+v, %v", stored, err)
1276 }
1277}
1278
1279// raceOnReadStore runs race() the FIRST time onGet is read, before the read
1280// returns — a competing writer landing at a chosen point in a walk.
1281type raceOnReadStore struct {
1282 release.Store
1283 onGet string
1284 race func(release.Store)
1285}
1286
1287func (r *raceOnReadStore) Get(ctx context.Context, key string) ([]byte, string, error) {
1288 if r.onGet != "" && key == r.onGet {
1289 r.onGet = ""
1290 r.race(r.Store)
1291 }
1292 return r.Store.Get(ctx, key)
1293}
1294
1295// A grant that lands mid-teardown has re-provisioned a live estate and
1296// recorded it. The final delete must not erase that record just because this
1297// walk read an older one.
1298func TestTeardownRefusesConcurrentRegrant(t *testing.T) {
1299 ctx := context.Background()
1300 inner := release.NewDirStore(t.TempDir())
1301 _, pub := testSealKeypair(t)
1302 p := NewMemProvisioner("eu-west-1")
1303 s := &raceOnReadStore{Store: inner}
1304 teardownFixture(t, s, pub, p)
1305
1306 // Fire on the config probe — the last read Teardown does before its
1307 // re-read guard, so the competing record is in place by the time the
1308 // guard looks.
1309 s.onGet = appconfig.Key("bab", "titogo")
1310 s.race = func(in release.Store) {
1311 other, etag, err := Get(ctx, in, "bab", "titogo")
1312 if err != nil {
1313 t.Error(err)
1314 return
1315 }
1316 other.Provisioned.KeyID = "MEMKEY99" // a fresh grant landed and recorded
1317 if _, err := Put(ctx, in, "bab", "titogo", other, etag); err != nil {
1318 t.Error(err)
1319 }
1320 }
1321
1322 if err := Teardown(ctx, s, p, nil, "bab", "titogo"); !errors.Is(err, release.ErrCASConflict) {
1323 t.Fatalf("teardown over a concurrent grant: %v, want ErrCASConflict", err)
1324 }
1325 stored, _, err := Get(ctx, inner, "bab", "titogo")
1326 if err != nil || stored == nil || stored.Provisioned == nil || stored.Provisioned.KeyID != "MEMKEY99" {
1327 t.Fatalf("the competing grant's record was erased: %+v, %v", stored, err)
1328 }
1329}
1330
1331// TestPurgeGuardSeesOurRecord pins the PARSE AGREEMENT between this package
1332// and release.PurgeApp. release cannot import objstore (objstore imports
1333// release), so PurgeApp re-parses `grant` and `provisioned` out of store.json
1334// with a local struct of its own. That guard is what stops a box's auto-purge
1335// from sweeping away the only record of a paid bucket and a live IAM user.
1336// Records written HERE, by the real writer, must trip it — for the finished
1337// store and for the crashed grant alike.
1338func TestPurgeGuardSeesOurRecord(t *testing.T) {
1339 ctx := context.Background()
1340 _, pub := testSealKeypair(t)
1341 relPub, _, err := ed25519.GenerateKey(rand.Reader)
1342 if err != nil {
1343 t.Fatal(err)
1344 }
1345 // Past the cooloff, so nothing but the store guard stands between the
1346 // purge and destruction.
1347 backdated := time.Now().Add(-2 * release.TrashCooloff)
1348
1349 t.Run("provisioned", func(t *testing.T) {
1350 s := release.NewDirStore(t.TempDir())
1351 p := NewMemProvisioner("eu-west-1")
1352 rec := rotateFixture(t, s, pub, p) // declare + Grant: a real provisioned record
1353 if rec.Provisioned == nil {
1354 t.Fatal("fixture produced no provisioned block")
1355 }
1356 // Drop the grant block so `provisioned` stands ALONE. A real record
1357 // carries both, which would let this subtest pass on the grant key and
1358 // leave Provisioned's json tag unpinned — the one thing this test
1359 // exists to catch.
1360 stored, etag, err := Get(ctx, s, "bab", "titogo")
1361 if err != nil || stored == nil {
1362 t.Fatalf("re-read: %+v, %v", stored, err)
1363 }
1364 stored.Grant = nil
1365 if _, err := Put(ctx, s, "bab", "titogo", stored, etag); err != nil {
1366 t.Fatal(err)
1367 }
1368 if err := release.TrashApp(ctx, s, "bab", "titogo", "paul@keymail.dev", backdated); err != nil {
1369 t.Fatal(err)
1370 }
1371 err = release.PurgeApp(ctx, s, relPub, "bab", "titogo", time.Now())
1372 if !errors.Is(err, release.ErrStoreProvisioned) {
1373 t.Fatalf("PurgeApp over a provisioned record = %v, want release.ErrStoreProvisioned", err)
1374 }
1375 if _, _, err := s.Get(ctx, Key("bab", "titogo")); err != nil {
1376 t.Fatalf("the refused purge destroyed store.json: %v", err)
1377 }
1378 })
1379
1380 t.Run("crashed-grant", func(t *testing.T) {
1381 s := release.NewDirStore(t.TempDir())
1382 p := NewMemProvisioner("eu-west-1")
1383 crashedGrant(t, s, p) // grant block, no provisioned block, live estate
1384 if err := release.TrashApp(ctx, s, "bab", "titogo", "paul@keymail.dev", backdated); err != nil {
1385 t.Fatal(err)
1386 }
1387 err := release.PurgeApp(ctx, s, relPub, "bab", "titogo", time.Now())
1388 if !errors.Is(err, release.ErrStoreProvisioned) {
1389 t.Fatalf("PurgeApp over a crashed grant = %v, want release.ErrStoreProvisioned", err)
1390 }
1391 })
1392
1393 // The other direction, so the guard cannot pass by refusing everything: a
1394 // declared-only record has no backend estate and is purged like any other
1395 // object. This is also the state Teardown leaves behind (it deletes the
1396 // record outright), which is how a torn-down app becomes purgeable.
1397 t.Run("declared-only-purges", func(t *testing.T) {
1398 s := release.NewDirStore(t.TempDir())
1399 declare(t, s, "bab", "titogo", "TITO_S3")
1400 if err := release.TrashApp(ctx, s, "bab", "titogo", "paul@keymail.dev", backdated); err != nil {
1401 t.Fatal(err)
1402 }
1403 if err := release.PurgeApp(ctx, s, relPub, "bab", "titogo", time.Now()); err != nil {
1404 t.Fatalf("PurgeApp over a declared-only record: %v", err)
1405 }
1406 if _, _, err := s.Get(ctx, Key("bab", "titogo")); !errors.Is(err, os.ErrNotExist) {
1407 t.Fatalf("store.json survived the purge: %v", err)
1408 }
1409 })
1410}
1411
1412// Source records WHY a grant is authorised (#381), so that an approval which
1413// is later withdrawn can be told apart from an operator's deliberate per-app
1414// grant — mail.GrantBlock.Source's reasoning, applied to storage.
1415func TestGrantRecordsItsSource(t *testing.T) {
1416 ctx := context.Background()
1417 _, pub := testSealKeypair(t)
1418 s := release.NewDirStore(t.TempDir())
1419 p := NewMemProvisioner("eu-west-1")
1420 declare(t, s, "bab", "titogo", "TITO_S3")
1421
1422 rec, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo",
1423 "op@example.com", GrantSourceAccountApproval, "", time.Now().UTC())
1424 if err != nil {
1425 t.Fatal(err)
1426 }
1427 if rec.Grant.Source != GrantSourceAccountApproval {
1428 t.Fatalf("grant source = %q, want %q", rec.Grant.Source, GrantSourceAccountApproval)
1429 }
1430 if rec.Grant.By != "op@example.com" {
1431 t.Fatalf("grant by = %q — an account-approved grant still names the approving operator", rec.Grant.By)
1432 }
1433 stored, _, err := Get(ctx, s, "bab", "titogo")
1434 if err != nil || stored.Grant.Source != GrantSourceAccountApproval {
1435 t.Fatalf("stored grant = %+v, %v", stored.Grant, err)
1436 }
1437}
1438
1439// A re-run repairs an estate whose authority was already established. Restating
1440// the source would let a repair rewrite the record's account of who authorised
1441// the spend — the same reason a re-run never re-chooses the bucket.
1442func TestGrantDoesNotRestateSourceOnRerun(t *testing.T) {
1443 ctx := context.Background()
1444 _, pub := testSealKeypair(t)
1445 s := release.NewDirStore(t.TempDir())
1446 p := NewMemProvisioner("eu-west-1")
1447 declare(t, s, "bab", "titogo", "TITO_S3")
1448
1449 if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo",
1450 "op@example.com", GrantSourceOperator, "", time.Now().UTC()); err != nil {
1451 t.Fatal(err)
1452 }
1453 // Re-run claiming a different authority; the first one must stand.
1454 rec, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo",
1455 "someone@example.com", GrantSourceAccountApproval, "", time.Now().UTC())
1456 if err != nil {
1457 t.Fatal(err)
1458 }
1459 if rec.Grant.Source != GrantSourceOperator || rec.Grant.By != "op@example.com" {
1460 t.Fatalf("a re-run rewrote the grant's authority: %+v", rec.Grant)
1461 }
1462}
1463
1464// grantBlockOnlyFixture leaves the record exactly as a walk that died mid-flight
1465// leaves it: the grant block written (Grant's step 1) and no provisioned block.
1466// A re-run then skips step 1's Put, so the next write it makes to the record is
1467// step 6's — the one these two tests race.
1468func grantBlockOnlyFixture(t *testing.T, s release.Store) {
1469 t.Helper()
1470 ctx := context.Background()
1471 declare(t, s, "bab", "titogo", "TITO_S3")
1472 rec, etag, err := Get(ctx, s, "bab", "titogo")
1473 if err != nil {
1474 t.Fatal(err)
1475 }
1476 rec.Grant = &GrantBlock{
1477 Backend: testDeployment().Backend,
1478 Bucket: MintedBucket(testDeployment(), "bab", "titogo"),
1479 At: time.Now().UTC(),
1480 By: "op",
1481 }
1482 if _, err := Put(ctx, s, "bab", "titogo", rec, etag); err != nil {
1483 t.Fatal(err)
1484 }
1485}
1486
1487// Grant's final Put has the same window Rotate's does — the console's declare
1488// door writes this very object, and under self-serve storage (#381) a member's
1489// own `carlos store create` drives the walk, so the window is ordinary rather
1490// than rare. Losing that CAS outright leaves the record naming no key (or an
1491// older one) while the app already holds the delivered credential, and the next
1492// rotate's reap then deletes the key actually in use — the outcome Rotate's own
1493// doc comment calls NOT survivable. So Grant retries against a re-read record,
1494// and a competitor's change to blocks it does not own must survive.
1495func TestGrantRetriesLostCAS(t *testing.T) {
1496 ctx := context.Background()
1497 priv, pub := testSealKeypair(t)
1498 inner := release.NewDirStore(t.TempDir())
1499 s := &racingStore{Store: inner}
1500 p := NewMemProvisioner("eu-west-1")
1501 grantBlockOnlyFixture(t, s)
1502
1503 // A competing writer touches a block this walk does not own — exactly what
1504 // a re-declare through the console's create door does.
1505 s.raceKeyOnce = Key("bab", "titogo")
1506 s.race = func(in release.Store) {
1507 other, etag, err := Get(ctx, in, "bab", "titogo")
1508 if err != nil {
1509 t.Fatal(err)
1510 }
1511 other.Declared.By = "someone-else@example.com"
1512 if _, err := Put(ctx, in, "bab", "titogo", other, etag); err != nil {
1513 t.Fatal(err)
1514 }
1515 }
1516
1517 rec, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC())
1518 if err != nil {
1519 t.Fatalf("grant lost its CAS and gave up: %v", err)
1520 }
1521 if rec.Provisioned == nil || rec.Provisioned.KeyID == "" {
1522 t.Fatalf("provisioned = %+v", rec.Provisioned)
1523 }
1524 if rec.Declared.By != "someone-else@example.com" {
1525 t.Fatalf("retry clobbered the competing write: Declared.By = %q", rec.Declared.By)
1526 }
1527 stored, _, err := Get(ctx, s, "bab", "titogo")
1528 if err != nil || stored.Declared.By != "someone-else@example.com" ||
1529 stored.Provisioned == nil || stored.Provisioned.KeyID != rec.Provisioned.KeyID {
1530 t.Fatalf("stored record = %+v / %+v, %v", stored.Declared, stored.Provisioned, err)
1531 }
1532 // The record and the app agree on the live key — the whole point.
1533 if got := openVar(t, priv, s, "bab", "titogo", "TITO_S3_ACCESS_KEY"); got != stored.Provisioned.KeyID {
1534 t.Fatalf("delivered access key = %q, record says %q", got, stored.Provisioned.KeyID)
1535 }
1536}
1537
1538// Grant-vs-grant is the case the retry must NOT paper over: a competing walk
1539// minted and DELIVERED a key of its own, so overwriting its record would strand
1540// that key exactly as a lost CAS would strand ours. The conflict is surfaced.
1541func TestGrantRefusesRetryAgainstAnotherProvision(t *testing.T) {
1542 ctx := context.Background()
1543 _, pub := testSealKeypair(t)
1544 inner := release.NewDirStore(t.TempDir())
1545 s := &racingStore{Store: inner}
1546 p := NewMemProvisioner("eu-west-1")
1547 grantBlockOnlyFixture(t, s)
1548
1549 s.raceKeyOnce = Key("bab", "titogo")
1550 s.race = func(in release.Store) {
1551 other, etag, err := Get(ctx, in, "bab", "titogo")
1552 if err != nil {
1553 t.Fatal(err)
1554 }
1555 other.Provisioned = &Provisioned{
1556 Bucket: other.Grant.Bucket,
1557 Region: "eu-west-1",
1558 IAMUser: UserName("bab", "titogo"),
1559 KeyID: "MEMKEY99", // another grant landed first, and delivered
1560 At: time.Now().UTC(),
1561 }
1562 if _, err := Put(ctx, in, "bab", "titogo", other, etag); err != nil {
1563 t.Fatal(err)
1564 }
1565 }
1566
1567 if _, err := Grant(ctx, s, p, testDeployment(), pub, "bab", "titogo", "op", GrantSourceOperator, "", time.Now().UTC()); !errors.Is(err, release.ErrCASConflict) {
1568 t.Fatalf("grant over a concurrent grant: %v, want ErrCASConflict", err)
1569 }
1570 stored, _, err := Get(ctx, s, "bab", "titogo")
1571 if err != nil || stored.Provisioned == nil || stored.Provisioned.KeyID != "MEMKEY99" {
1572 t.Fatalf("the competing grant was clobbered: %+v, %v", stored.Provisioned, err)
1573 }
1574}
1575