amadan

carlos / platform Public

Clone 
git clone https://amadan.net/carlos/platform

Plain git — no account needed to clone.

Download

Download this file

1package console
2
3import (
4 "context"
5 "encoding/json"
6 "errors"
7 "fmt"
8 "net/http"
9 "strings"
10 "sync"
11 "testing"
12 "time"
13
14 "github.com/carlosframework/platform/internal/objstore"
15 "github.com/carlosframework/platform/internal/release"
16 "github.com/carlosframework/platform/internal/svcrole"
17)
18
19func storeProvisionURL(fx *consoleFixture, app string) string {
20 return storeURL(fx, app) + "/provision"
21}
22
23func approveStoreAccount(t *testing.T, fx *consoleFixture, by string) {
24 t.Helper()
25 putEntitlement(t, fx, fx.sqid, &objstore.Entitlement{
26 Applied: &objstore.Applied{By: "paul@keymail.dev", At: time.Now().Add(-time.Minute)},
27 Decision: &objstore.Decision{Approved: true, By: by, At: time.Now()},
28 })
29}
30
31type pausedBucketProvisioner struct {
32 objstore.Provisioner
33 entered chan struct{}
34 release chan struct{}
35 once sync.Once
36}
37
38func (p *pausedBucketProvisioner) EnsureBucket(ctx context.Context, name string) (string, error) {
39 p.once.Do(func() {
40 close(p.entered)
41 select {
42 case <-p.release:
43 case <-ctx.Done():
44 }
45 })
46 return p.Provisioner.EnsureBucket(ctx, name)
47}
48
49type provisionResult struct {
50 status int
51 out map[string]any
52}
53
54func provisionAsync(t *testing.T, fx *consoleFixture, token string) <-chan provisionResult {
55 t.Helper()
56 ch := make(chan provisionResult, 1)
57 go func() {
58 defer close(ch)
59 status, raw := postRaw(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token)
60 out := map[string]any{}
61 if err := json.Unmarshal([]byte(raw), &out); err != nil {
62 out["raw"] = raw
63 }
64 ch <- provisionResult{status: status, out: out}
65 }()
66 return ch
67}
68
69func waitPaused(t *testing.T, p *pausedBucketProvisioner) {
70 t.Helper()
71 select {
72 case <-p.entered:
73 case <-time.After(3 * time.Second):
74 t.Fatal("provision did not reach the paused backend")
75 }
76}
77
78func TestAPIStoreProvisionNeedsAnApprovedEntitlement(t *testing.T) {
79 fx, fake, token, _ := storeArmed(t, "code-store-provision-unapproved")
80 declareStore(t, fx, token)
81
82 status, _ := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token)
83 if status < 400 {
84 t.Fatalf("provision without approval = %d, want a refusal", status)
85 }
86 untouched(t, fake, "an unapproved provision")
87 if rec := storedRecord(t, fx, "titogo"); rec.Grant != nil {
88 t.Fatalf("an unapproved provision wrote a grant: %+v", rec.Grant)
89 }
90}
91
92func TestAPIStoreProvisionWithoutABackendDoesNotGrant(t *testing.T) {
93 fx, fake, token, _ := storeArmed(t, "code-store-provision-nobackend")
94 declareStore(t, fx, token)
95 approveStoreAccount(t, fx, "operator@keymail.dev")
96 fx.srv.cfg.StoreProvisioner = nil
97
98 status, _ := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token)
99 if status != http.StatusServiceUnavailable {
100 t.Fatalf("provision without a backend = %d, want 503", status)
101 }
102 untouched(t, fake, "a backend-less provision")
103 if rec := storedRecord(t, fx, "titogo"); rec.Grant != nil {
104 t.Fatalf("a backend-less provision wrote a grant: %+v", rec.Grant)
105 }
106}
107
108func TestAPIStoreProvisionAttributesAccountApproval(t *testing.T) {
109 fx, fake, token, priv := storeArmed(t, "code-store-provision-happy")
110 declareStore(t, fx, token)
111 approveStoreAccount(t, fx, "operator@keymail.dev")
112 fx.srv.cfg.StoreOperators = nil
113
114 status, out := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token)
115 if status != http.StatusOK {
116 t.Fatalf("provision = %d %v, want 200", status, out)
117 }
118 rec := storedRecord(t, fx, "titogo")
119 if rec.Provisioned == nil || rec.Grant == nil {
120 t.Fatalf("provisioned record = %+v", rec)
121 }
122 if rec.Grant.Source != objstore.GrantSourceAccountApproval || rec.Grant.By != "operator@keymail.dev" {
123 t.Fatalf("grant = %+v, want the account approval source and deciding operator", rec.Grant)
124 }
125 u := fake.Users[rec.Provisioned.IAMUser]
126 secret, ok := u.Keys[rec.Provisioned.KeyID]
127 if !ok {
128 t.Fatalf("recorded key %q is absent from backend keys %v", rec.Provisioned.KeyID, u.Keys)
129 }
130 if got := openSecret(t, priv, appVars(t, fx, "titogo")["CARLOS_STORE_SECRET_KEY"]); got != secret {
131 t.Fatalf("delivered secret does not match the recorded backend key")
132 }
133 if entitlementBlock(t, out)["decision"] == nil {
134 t.Fatalf("provision reply omitted the entitlement: %v", out)
135 }
136}
137
138func TestAPIStoreProvisionCompletedStoreIsANoOp(t *testing.T) {
139 fx, fake, token, _ := storeArmed(t, "code-store-provision-noop")
140 declareStore(t, fx, token)
141 approveStoreAccount(t, fx, "operator@keymail.dev")
142 if status, out := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK {
143 t.Fatalf("first provision = %d %v", status, out)
144 }
145 before := storedRecord(t, fx, "titogo").Provisioned.KeyID
146 if status, out := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK {
147 t.Fatalf("second provision = %d %v", status, out)
148 }
149 after := storedRecord(t, fx, "titogo").Provisioned.KeyID
150 if after != before || len(fake.Users["carlos-store-bab-titogo"].Keys) != 1 {
151 t.Fatalf("no-op changed key %q to %q or minted extras: %v", before, after, fake.Users)
152 }
153}
154
155func TestAPIStoreProvisionRetriesAFailedGrantWithoutChangingBucket(t *testing.T) {
156 fx, fake, token, _ := storeArmed(t, "code-store-provision-retry")
157 declareStore(t, fx, token)
158 approveStoreAccount(t, fx, "operator@keymail.dev")
159 fake.FailNext["EnsureBucket"] = errors.New("backend detail must stay private")
160
161 status, out := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token)
162 if status != http.StatusBadGateway {
163 t.Fatalf("failed provision = %d %v, want 502", status, out)
164 }
165 if strings.Contains(str(t, out, "error"), "backend detail") {
166 t.Fatalf("provision response leaked the backend failure: %v", out)
167 }
168 failed := storedRecord(t, fx, "titogo")
169 if failed.Grant == nil || failed.Provisioned != nil {
170 t.Fatalf("failed provision record = %+v, want grant-only", failed)
171 }
172 bucket := failed.Grant.Bucket
173 if status, out := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token); status != http.StatusOK {
174 t.Fatalf("retry provision = %d %v, want 200", status, out)
175 }
176 got := storedRecord(t, fx, "titogo")
177 if got.Provisioned == nil || got.Provisioned.Bucket != bucket || got.Grant.Bucket != bucket {
178 t.Fatalf("retry changed or failed to provision bucket %q: %+v", bucket, got)
179 }
180}
181
182func TestAPIStoreProvisionSerializesAndRereadsEntitlement(t *testing.T) {
183 fx, fake, token, _ := storeArmed(t, "code-store-provision-revoke")
184 declareStore(t, fx, token)
185 approveStoreAccount(t, fx, "operator@keymail.dev")
186 paused := &pausedBucketProvisioner{Provisioner: fake, entered: make(chan struct{}), release: make(chan struct{})}
187 fx.srv.cfg.StoreProvisioner = paused
188 defer func() {
189 select {
190 case <-paused.release:
191 default:
192 close(paused.release)
193 }
194 }()
195
196 first := provisionAsync(t, fx, token)
197 waitPaused(t, paused)
198 second := provisionAsync(t, fx, token)
199 waitStoreProvisionQueued(t, fx)
200 e, etag, err := objstore.GetEntitlement(context.Background(), fx.rel, fx.sqid)
201 if err != nil {
202 t.Fatal(err)
203 }
204 e.Decision = &objstore.Decision{Approved: false, By: "operator@keymail.dev", At: time.Now(), Reason: "capacity review"}
205 if _, err := objstore.PutEntitlement(context.Background(), fx.rel, fx.sqid, e, etag); err != nil {
206 t.Fatal(err)
207 }
208 close(paused.release)
209 if got := <-first; got.status != http.StatusOK {
210 t.Fatalf("first provision = %d %v, want 200", got.status, got.out)
211 }
212 if got := <-second; got.status < 400 {
213 t.Fatalf("queued provision after revocation = %d %v, want a refusal", got.status, got.out)
214 }
215 status, out := getJSON(t, storeURL(fx, "titogo"), token)
216 if status != http.StatusOK || block(t, entitlementBlock(t, out), "decision")["approved"] != false {
217 t.Fatalf("declined status = %d %v", status, out)
218 }
219 rec := storedRecord(t, fx, "titogo")
220 if rec.Provisioned == nil || len(fake.Users[rec.Provisioned.IAMUser].Keys) != 1 {
221 t.Fatalf("serialized provisions left the wrong estate: record=%+v users=%v", rec, fake.Users)
222 }
223}
224
225func TestAPIStoreDeclareSamePrefixDuringProvisionIsNoOp(t *testing.T) {
226 fx, fake, token, _ := storeArmed(t, "code-store-provision-declare-race")
227 declareStore(t, fx, token)
228 approveStoreAccount(t, fx, "operator@keymail.dev")
229 paused := &pausedBucketProvisioner{Provisioner: fake, entered: make(chan struct{}), release: make(chan struct{})}
230 fx.srv.cfg.StoreProvisioner = paused
231 defer func() {
232 select {
233 case <-paused.release:
234 default:
235 close(paused.release)
236 }
237 }()
238 result := provisionAsync(t, fx, token)
239 waitPaused(t, paused)
240
241 before, beforeETag, err := objstore.Get(context.Background(), fx.rel, release.OpsAccount, "titogo")
242 if err != nil {
243 t.Fatal(err)
244 }
245 status, out := postJSON(t, storeURL(fx, "titogo"), map[string]any{}, token)
246 if status != http.StatusOK {
247 t.Fatalf("same-prefix declare = %d %v, want 200", status, out)
248 }
249 after, afterETag, err := objstore.Get(context.Background(), fx.rel, release.OpsAccount, "titogo")
250 if err != nil {
251 t.Fatal(err)
252 }
253 if afterETag != beforeETag || after.Declared.By != before.Declared.By || !after.Declared.At.Equal(before.Declared.At) {
254 t.Fatalf("same-prefix declare changed record: etag %q -> %q, declared %+v -> %+v", beforeETag, afterETag, before.Declared, after.Declared)
255 }
256 close(paused.release)
257 if got := <-result; got.status != http.StatusOK {
258 t.Fatalf("provision after no-op declare = %d %v, want 200", got.status, got.out)
259 }
260}
261
262func TestAPIStoreAdminServiceDeclarationKeepsServiceAttribution(t *testing.T) {
263 fx, _, _, _ := storeArmed(t, "code-store-provision-service")
264 admin := serviceToken(t, fx.st, OpsAccountID, "storebot", "", svcrole.Admin)
265
266 status, out := postJSON(t, storeURL(fx, "titogo"), map[string]any{}, admin)
267 if status != http.StatusOK {
268 t.Fatalf("admin service declare = %d %v, want 200", status, out)
269 }
270 if got := storedRecord(t, fx, "titogo").Declared.By; got != "service:bab/storebot" {
271 t.Fatalf("declared.by = %q, want service attribution", got)
272 }
273}
274
275func waitStoreProvisionQueued(t *testing.T, fx *consoleFixture) {
276 t.Helper()
277 deadline := time.Now().Add(3 * time.Second)
278 for {
279 fx.srv.storeWalks.mu.Lock()
280 e := fx.srv.storeWalks.m[storeWalkKey(fx.sqid, "titogo", "")]
281 refs := 0
282 if e != nil {
283 refs = e.refs
284 }
285 fx.srv.storeWalks.mu.Unlock()
286 if refs == 2 {
287 return
288 }
289 if time.Now().After(deadline) {
290 t.Fatalf("second provision never queued: refs=%d", refs)
291 }
292 time.Sleep(time.Millisecond)
293 }
294}
295
296func TestAPIStoreConcurrentProvisionsMintOneDeliveredKey(t *testing.T) {
297 fx, fake, token, priv := storeArmed(t, "code-store-provision-two")
298 declareStore(t, fx, token)
299 approveStoreAccount(t, fx, "operator@keymail.dev")
300 paused := &pausedBucketProvisioner{Provisioner: fake, entered: make(chan struct{}), release: make(chan struct{})}
301 fx.srv.cfg.StoreProvisioner = paused
302 defer func() {
303 select {
304 case <-paused.release:
305 default:
306 close(paused.release)
307 }
308 }()
309 first := provisionAsync(t, fx, token)
310 waitPaused(t, paused)
311 second := provisionAsync(t, fx, token)
312 waitStoreProvisionQueued(t, fx)
313 close(paused.release)
314 for _, result := range []<-chan provisionResult{first, second} {
315 select {
316 case got := <-result:
317 if got.status != http.StatusOK {
318 t.Fatalf("provision = %d %v", got.status, got.out)
319 }
320 case <-time.After(3 * time.Second):
321 t.Fatal("provision did not finish")
322 }
323 }
324 rec := storedRecord(t, fx, "titogo")
325 if rec.Provisioned == nil {
326 t.Fatal("store was not provisioned")
327 }
328 keys := fake.Users[rec.Provisioned.IAMUser].Keys
329 if rec.Provisioned.KeyID != "MEMKEY1" || len(keys) != 1 {
330 t.Fatalf("wanted exactly one mint: record=%+v keys=%v", rec.Provisioned, keys)
331 }
332 vars := appVars(t, fx, "titogo")
333 if openSecret(t, priv, vars["CARLOS_STORE_ACCESS_KEY"]) != rec.Provisioned.KeyID {
334 t.Fatal("delivered key differs from record")
335 }
336 if openSecret(t, priv, vars["CARLOS_STORE_SECRET_KEY"]) != keys[rec.Provisioned.KeyID] {
337 t.Fatal("delivered secret differs from backend")
338 }
339}
340
341func TestAPIStoreDeclareReportsEntitlementWithoutProvisioning(t *testing.T) {
342 for _, approved := range []bool{false, true} {
343 for _, backend := range []bool{false, true} {
344 t.Run(fmt.Sprintf("approved=%t/backend=%t", approved, backend), func(t *testing.T) {
345 fx, fake, token, _ := storeArmed(t, "code-store-declare-entitlement")
346 if approved {
347 approveStoreAccount(t, fx, "operator@keymail.dev")
348 }
349 if !backend {
350 fx.srv.cfg.StoreProvisioner = nil
351 fx.srv.cfg.StoreDeployment = objstore.Deployment{}
352 }
353 status, out := postJSON(t, storeURL(fx, "titogo"), map[string]any{}, token)
354 if status != http.StatusOK {
355 t.Fatalf("declare = %d %v", status, out)
356 }
357 if _, present := out["entitlement"]; !present {
358 t.Fatal("reply omitted entitlement")
359 }
360 if approved {
361 if !block(t, entitlementBlock(t, out), "decision")["approved"].(bool) {
362 t.Fatal("approval missing")
363 }
364 } else if out["entitlement"] != nil {
365 t.Fatalf("unexpected entitlement: %v", out)
366 }
367 if (out["backend"] != "") != backend {
368 t.Fatalf("backend mismatch: %v", out)
369 }
370 if out["grant"] != nil || out["provisioned"] != nil {
371 t.Fatalf("declare minted estate: %v", out)
372 }
373 untouched(t, fake, "declare")
374 })
375 }
376 }
377}
378
379func TestAPIStoreProvisionMemberAndServiceGates(t *testing.T) {
380 fx, fake, token, _ := storeArmed(t, "code-store-provision-gates")
381 declareStore(t, fx, token)
382 approveStoreAccount(t, fx, "operator@keymail.dev")
383 stranger := cliToken(t, fx, "stranger@keymail.dev", "code-store-provision-stranger")
384 status, body := postRaw(t, storeProvisionURL(fx, "titogo"), map[string]any{}, stranger)
385 if status != http.StatusNotFound || body != "404 page not found\n" {
386 t.Fatalf("stranger = %d %q", status, body)
387 }
388 for _, role := range []svcrole.Role{svcrole.Publish, svcrole.Instance, svcrole.Operate} {
389 svc := serviceToken(t, fx.st, OpsAccountID, "denied-"+string(role), "", role)
390 status, _ := postRaw(t, storeProvisionURL(fx, "titogo"), map[string]any{}, svc)
391 if status != http.StatusForbidden {
392 t.Fatalf("service %s = %d", role, status)
393 }
394 }
395 untouched(t, fake, "refused provision")
396 admin := serviceToken(t, fx.st, OpsAccountID, "store-admin", "titogo", svcrole.Admin)
397 status, out := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, admin)
398 if status != http.StatusOK {
399 t.Fatalf("admin provision = %d %v", status, out)
400 }
401 if storedRecord(t, fx, "titogo").Grant.By != "operator@keymail.dev" {
402 t.Fatal("service replaced approving actor")
403 }
404 auditedBy(t, fx, "titogo", "store-provision", "service:bab/store-admin", "provisioned")
405}
406
407func TestAPIStoreProvisionRequiresDeclarationAndSealKey(t *testing.T) {
408 fx, fake, token, _ := storeArmed(t, "code-store-provision-preconditions")
409 approveStoreAccount(t, fx, "operator@keymail.dev")
410 status, out := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token)
411 if status != http.StatusConflict {
412 t.Fatalf("undeclared = %d %v", status, out)
413 }
414 declareStore(t, fx, token)
415 fx.srv.cfg.SecretsPub = ""
416 status, out = postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token)
417 if status != http.StatusServiceUnavailable {
418 t.Fatalf("unsealable = %d %v", status, out)
419 }
420 untouched(t, fake, "unsealable provision")
421}
422
423func TestAPIStoreProvisionCannotAdoptABucket(t *testing.T) {
424 fx, fake, token, _ := storeArmed(t, "code-store-provision-no-adoption")
425 declareStore(t, fx, token)
426 approveStoreAccount(t, fx, "operator@keymail.dev")
427 status, out := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{"bucket": "someone-elses-data"}, token)
428 if status != http.StatusOK && status != http.StatusBadRequest {
429 t.Fatalf("provision with adoption input = %d %v", status, out)
430 }
431 if _, exists := fake.Buckets["someone-elses-data"]; exists {
432 t.Fatal("member provision adopted caller bucket")
433 }
434 rec := storedRecord(t, fx, "titogo")
435 if rec.Grant != nil && rec.Grant.Bucket != objstore.MintedBucket(fx.srv.cfg.StoreDeployment, fx.sqid, "titogo") {
436 t.Fatalf("grant chose caller bucket: %+v", rec.Grant)
437 }
438}
439
440func TestAPIStoreProvisionFailsClosedOnUnreadableApproval(t *testing.T) {
441 fx, fake, token, _ := storeArmed(t, "code-store-provision-unreadable")
442 declareStore(t, fx, token)
443 if _, err := fx.rel.Put(context.Background(), objstore.EntitlementKey(fx.sqid), []byte("private backend details"), ""); err != nil {
444 t.Fatal(err)
445 }
446 status, out := postJSON(t, storeProvisionURL(fx, "titogo"), map[string]any{}, token)
447 if status != http.StatusBadGateway {
448 t.Fatalf("unreadable approval = %d %v", status, out)
449 }
450 if strings.Contains(str(t, out, "error"), "private") {
451 t.Fatal("approval error leaked backend details")
452 }
453 untouched(t, fake, "unreadable approval")
454}
455