| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "errors" |
| 7 | "net/http" |
| 8 | "net/http/httptest" |
| 9 | "os" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/carlosframework/platform/internal/objstore" |
| 14 | "github.com/carlosframework/platform/internal/release" |
| 15 | ) |
| 16 | |
| 17 | func TestSeedOwnsStore(t *testing.T) { |
| 18 | id := "sd_aaaaaaaaaaaaaaaaaaaaaaaaaa" |
| 19 | for _, c := range []struct { |
| 20 | rec *objstore.Record |
| 21 | want bool |
| 22 | }{ |
| 23 | {nil, true}, // no store: this seed may make one |
| 24 | {&objstore.Record{Declared: &objstore.Declared{}}, true}, // declared only: no estate yet |
| 25 | {&objstore.Record{Declared: &objstore.Declared{Seed: id}, Grant: &objstore.GrantBlock{}}, true}, |
| 26 | {&objstore.Record{Declared: &objstore.Declared{Seed: "sd_bbbbbbbbbbbbbbbbbbbbbbbbbb"}, Grant: &objstore.GrantBlock{}}, false}, |
| 27 | {&objstore.Record{Declared: &objstore.Declared{}, Provisioned: &objstore.Provisioned{}}, false}, |
| 28 | {&objstore.Record{Grant: &objstore.GrantBlock{}}, false}, // estate nobody declared: not this seed's |
| 29 | } { |
| 30 | if got := seedOwnsStore(c.rec, id); got != c.want { |
| 31 | t.Errorf("seedOwnsStore(%+v) = %v, want %v", c.rec, got, c.want) |
| 32 | } |
| 33 | } |
| 34 | } |
| 35 | |
| 36 | const ( |
| 37 | seedA = "sd_aaaaaaaaaaaaaaaaaaaaaaaaaa" |
| 38 | seedB = "sd_bbbbbbbbbbbbbbbbbbbbbbbbbb" |
| 39 | ) |
| 40 | |
| 41 | // seededStoreFixture is privateStorageFixture with a neutral app, shop, |
| 42 | // whose canary environment the seeded tests provision. |
| 43 | func seededStoreFixture(t *testing.T, code string) (*consoleFixture, *objstore.MemProvisioner, *Account) { |
| 44 | t.Helper() |
| 45 | fx, fake, _, _ := privateStorageFixture(t, code) |
| 46 | if _, err := fx.st.CreateAppIn(OpsAccountID, "shop", "paul@keymail.dev", time.Now()); err != nil { |
| 47 | t.Fatal(err) |
| 48 | } |
| 49 | acct, err := fx.st.AccountByID(OpsAccountID) |
| 50 | if err != nil { |
| 51 | t.Fatal(err) |
| 52 | } |
| 53 | return fx, fake, acct |
| 54 | } |
| 55 | |
| 56 | func putCanaryStore(t *testing.T, fx *consoleFixture, rec *objstore.Record) { |
| 57 | t.Helper() |
| 58 | if _, err := objstore.PutIn(context.Background(), fx.rel, release.OpsAccount, "shop", "canary", rec, ""); err != nil { |
| 59 | t.Fatal(err) |
| 60 | } |
| 61 | } |
| 62 | |
| 63 | func canaryStore(t *testing.T, fx *consoleFixture) *objstore.Record { |
| 64 | t.Helper() |
| 65 | rec, _, err := objstore.GetIn(context.Background(), fx.rel, release.OpsAccount, "shop", "canary") |
| 66 | if err != nil { |
| 67 | t.Fatal(err) |
| 68 | } |
| 69 | return rec |
| 70 | } |
| 71 | |
| 72 | // A store that already holds estate and is not this seed's is refused |
| 73 | // under the store lock, before any declaration or grant: a concurrent |
| 74 | // `carlos store create --environment canary`, or another seed, must never |
| 75 | // hand a seeded instance a store that is not new and empty. Nothing is |
| 76 | // answered on w; the caller speaks the refusal. |
| 77 | func TestInstanceStoreProvisionForRefusesAStoreThatIsNotThisSeeds(t *testing.T) { |
| 78 | at := time.Now().UTC().Add(-time.Hour) |
| 79 | for name, planted := range map[string]*objstore.Record{ |
| 80 | "another seed's grant": { |
| 81 | Declared: &objstore.Declared{EnvPrefix: "CARLOS_STORE", At: at, By: "alice@keymail.dev", Seed: seedB}, |
| 82 | Grant: &objstore.GrantBlock{Backend: "mem", Bucket: "carlos-bab-shop-canary", Source: objstore.GrantSourceAccountApproval, At: at, By: "operator@keymail.dev"}, |
| 83 | }, |
| 84 | "an ordinary store's grant": { |
| 85 | Declared: &objstore.Declared{EnvPrefix: "CARLOS_STORE", At: at, By: "alice@keymail.dev"}, |
| 86 | Grant: &objstore.GrantBlock{Backend: "mem", Bucket: "carlos-bab-shop-canary", Source: objstore.GrantSourceAccountApproval, At: at, By: "operator@keymail.dev"}, |
| 87 | }, |
| 88 | "an ordinary provisioned store": { |
| 89 | Declared: &objstore.Declared{EnvPrefix: "CARLOS_STORE", At: at, By: "alice@keymail.dev"}, |
| 90 | Provisioned: &objstore.Provisioned{Bucket: "carlos-bab-shop-canary", Region: "eu-west-1", IAMUser: "carlos-store-bab-shop-canary", At: at}, |
| 91 | }, |
| 92 | } { |
| 93 | t.Run(name, func(t *testing.T) { |
| 94 | fx, fake, acct := seededStoreFixture(t, "code-seed-taken") |
| 95 | putCanaryStore(t, fx, planted) |
| 96 | w := httptest.NewRecorder() |
| 97 | got, ok, err := fx.srv.instanceStoreProvisionFor(context.Background(), w, acct, "shop", "canary", "alice@keymail.dev", seedA) |
| 98 | if ok || !errors.Is(err, errSeedStoreTaken) { |
| 99 | t.Fatalf("provision = %v, %v; want errSeedStoreTaken", ok, err) |
| 100 | } |
| 101 | if got == nil || storeBucketName(got) != "carlos-bab-shop-canary" { |
| 102 | t.Fatalf("returned record = %+v, want the store that is there, for the caller to name", got) |
| 103 | } |
| 104 | if w.Body.Len() != 0 { |
| 105 | t.Fatalf("answered %d %q, want nothing (the caller speaks this refusal)", w.Code, w.Body.String()) |
| 106 | } |
| 107 | if len(fake.Buckets) != 0 || len(fake.Users) != 0 { |
| 108 | t.Fatalf("backend buckets %v, users %v; want no grant walk", fake.Buckets, fake.Users) |
| 109 | } |
| 110 | after := canaryStore(t, fx) |
| 111 | if after.Declared == nil || after.Declared.Seed != planted.Declared.Seed || !after.Declared.At.Equal(at) { |
| 112 | t.Fatalf("declaration = %+v, want the planted one untouched", after.Declared) |
| 113 | } |
| 114 | }) |
| 115 | } |
| 116 | } |
| 117 | |
| 118 | // With no store, the seed declares one marked with its id and provisions |
| 119 | // it; a retry of the same seed then resumes that store. |
| 120 | func TestInstanceStoreProvisionForMarksANewStoreWithTheSeed(t *testing.T) { |
| 121 | fx, fake, acct := seededStoreFixture(t, "code-seed-new") |
| 122 | for range 2 { |
| 123 | w := httptest.NewRecorder() |
| 124 | out, ok, err := fx.srv.instanceStoreProvisionFor(context.Background(), w, acct, "shop", "canary", "alice@keymail.dev", seedA) |
| 125 | if !ok || err != nil { |
| 126 | t.Fatalf("provision = %v, %v (%d %s)", ok, err, w.Code, w.Body.String()) |
| 127 | } |
| 128 | if out.Provisioned == nil || out.Declared == nil || out.Declared.Seed != seedA { |
| 129 | t.Fatalf("store = %+v, want a provisioned store declared by %s", out, seedA) |
| 130 | } |
| 131 | } |
| 132 | if _, ok := fake.Buckets["carlos-bab-shop-canary"]; !ok || len(fake.Buckets) != 1 { |
| 133 | t.Fatalf("backend buckets = %v, want the one canary bucket", fake.Buckets) |
| 134 | } |
| 135 | } |
| 136 | |
| 137 | // A declared-only record holds no estate, so a seed takes it over: the |
| 138 | // declaration is rewritten for this seed (keeping no other seed's mark) |
| 139 | // and the store is provisioned. |
| 140 | func TestInstanceStoreProvisionForRedeclaresADeclaredOnlyRecord(t *testing.T) { |
| 141 | for name, decl := range map[string]*objstore.Declared{ |
| 142 | "another seed's": {EnvPrefix: "CARLOS_STORE", At: time.Now().UTC().Add(-time.Hour), By: "alice@keymail.dev", Seed: seedB}, |
| 143 | "an ordinary": {EnvPrefix: "CARLOS_STORE", At: time.Now().UTC().Add(-time.Hour), By: "alice@keymail.dev"}, |
| 144 | } { |
| 145 | t.Run(name, func(t *testing.T) { |
| 146 | fx, _, acct := seededStoreFixture(t, "code-seed-redeclare") |
| 147 | putCanaryStore(t, fx, &objstore.Record{Declared: decl}) |
| 148 | w := httptest.NewRecorder() |
| 149 | out, ok, err := fx.srv.instanceStoreProvisionFor(context.Background(), w, acct, "shop", "canary", "bob@keymail.dev", seedA) |
| 150 | if !ok || err != nil { |
| 151 | t.Fatalf("provision = %v, %v (%d %s)", ok, err, w.Code, w.Body.String()) |
| 152 | } |
| 153 | if out.Declared == nil || out.Declared.Seed != seedA || out.Declared.By != "bob@keymail.dev" || out.Provisioned == nil { |
| 154 | t.Fatalf("store = %+v / %+v, want provisioned and declared by %s", out, out.Declared, seedA) |
| 155 | } |
| 156 | }) |
| 157 | } |
| 158 | } |
| 159 | |
| 160 | // takeCanaryBucketName plants another store of shop that names canary's |
| 161 | // bucket, so canary's first grant fails with ErrStoreNameTaken before it |
| 162 | // writes a grant block. |
| 163 | func takeCanaryBucketName(t *testing.T, fx *consoleFixture) { |
| 164 | t.Helper() |
| 165 | at := time.Now().UTC() |
| 166 | rec := &objstore.Record{ |
| 167 | Declared: &objstore.Declared{EnvPrefix: "CARLOS_STORE", At: at, By: "alice@keymail.dev"}, |
| 168 | Grant: &objstore.GrantBlock{Backend: "mem", Bucket: "carlos-bab-shop-canary", Source: objstore.GrantSourceAccountApproval, At: at, By: "operator@keymail.dev"}, |
| 169 | } |
| 170 | if _, err := objstore.PutIn(context.Background(), fx.rel, release.OpsAccount, "shop", "staging", rec, ""); err != nil { |
| 171 | t.Fatal(err) |
| 172 | } |
| 173 | } |
| 174 | |
| 175 | func canaryRaw(t *testing.T, fx *consoleFixture) []byte { |
| 176 | t.Helper() |
| 177 | raw, _, err := fx.rel.Get(context.Background(), objstore.KeyIn(release.OpsAccount, "shop", "canary")) |
| 178 | if errors.Is(err, os.ErrNotExist) { |
| 179 | return nil |
| 180 | } |
| 181 | if err != nil { |
| 182 | t.Fatal(err) |
| 183 | } |
| 184 | return raw |
| 185 | } |
| 186 | |
| 187 | // A seed whose grant fails before writing its block leaves no record |
| 188 | // marked with its id. Another path could otherwise grant that record and |
| 189 | // keep the mark, and the seed's finish would tear down a store it never |
| 190 | // made. Here the seed created the record, so the record is gone. |
| 191 | func TestInstanceStoreProvisionForRollsBackAFreshDeclaration(t *testing.T) { |
| 192 | fx, fake, acct := seededStoreFixture(t, "code-seed-rollback-fresh") |
| 193 | takeCanaryBucketName(t, fx) |
| 194 | w := httptest.NewRecorder() |
| 195 | _, ok, err := fx.srv.instanceStoreProvisionFor(context.Background(), w, acct, "shop", "canary", "alice@keymail.dev", seedA) |
| 196 | if ok || err != nil || w.Code != http.StatusConflict { |
| 197 | t.Fatalf("provision = %v, %v (%d %s); want the name-taken refusal answered", ok, err, w.Code, w.Body.String()) |
| 198 | } |
| 199 | if raw := canaryRaw(t, fx); raw != nil { |
| 200 | t.Fatalf("canary record = %s, want none: the seed's declaration must not outlive its failed grant", raw) |
| 201 | } |
| 202 | if len(fake.Buckets) != 0 { |
| 203 | t.Fatalf("backend buckets = %v, want none", fake.Buckets) |
| 204 | } |
| 205 | } |
| 206 | |
| 207 | // Here the seed re-declared a record that was there before it, so the |
| 208 | // record goes back to exactly the bytes it had. |
| 209 | func TestInstanceStoreProvisionForRestoresARedeclaredRecord(t *testing.T) { |
| 210 | for name, decl := range map[string]*objstore.Declared{ |
| 211 | "another seed's": {EnvPrefix: "SHOP_FILES", At: time.Now().UTC().Add(-time.Hour), By: "alice@keymail.dev", Seed: seedB}, |
| 212 | "an ordinary": {EnvPrefix: "SHOP_FILES", At: time.Now().UTC().Add(-time.Hour), By: "alice@keymail.dev"}, |
| 213 | } { |
| 214 | t.Run(name, func(t *testing.T) { |
| 215 | fx, _, acct := seededStoreFixture(t, "code-seed-rollback-redeclare") |
| 216 | putCanaryStore(t, fx, &objstore.Record{Declared: decl}) |
| 217 | takeCanaryBucketName(t, fx) |
| 218 | before := canaryRaw(t, fx) |
| 219 | w := httptest.NewRecorder() |
| 220 | _, ok, err := fx.srv.instanceStoreProvisionFor(context.Background(), w, acct, "shop", "canary", "bob@keymail.dev", seedA) |
| 221 | if ok || err != nil || w.Code != http.StatusConflict { |
| 222 | t.Fatalf("provision = %v, %v (%d %s); want the name-taken refusal answered", ok, err, w.Code, w.Body.String()) |
| 223 | } |
| 224 | if after := canaryRaw(t, fx); !bytes.Equal(after, before) { |
| 225 | t.Fatalf("canary record after the failed seed:\n%s\nwant byte-identical to before:\n%s", after, before) |
| 226 | } |
| 227 | }) |
| 228 | } |
| 229 | } |
| 230 | |
| 231 | // A grant that wrote its block before failing has made estate for this |
| 232 | // seed: the record keeps the block and the mark, for the seed's finish. |
| 233 | func TestInstanceStoreProvisionForKeepsAGrantItWrote(t *testing.T) { |
| 234 | fx, fake, acct := seededStoreFixture(t, "code-seed-keep-grant") |
| 235 | fake.FailNext = map[string]error{"EnsureUser": errors.New("iam is down")} |
| 236 | w := httptest.NewRecorder() |
| 237 | _, ok, err := fx.srv.instanceStoreProvisionFor(context.Background(), w, acct, "shop", "canary", "alice@keymail.dev", seedA) |
| 238 | if ok || err != nil || w.Code != http.StatusBadGateway { |
| 239 | t.Fatalf("provision = %v, %v (%d %s); want the walk failure answered", ok, err, w.Code, w.Body.String()) |
| 240 | } |
| 241 | rec := canaryStore(t, fx) |
| 242 | if rec == nil || rec.Grant == nil || rec.Declared == nil || rec.Declared.Seed != seedA { |
| 243 | t.Fatalf("canary record = %+v, want the grant block and the seed's mark kept", rec) |
| 244 | } |
| 245 | } |
| 246 | |
| 247 | // A seed that wrote its grant block and then aborted leaves a store marked |
| 248 | // with its id. An ordinary create sharing that environment adopts the |
| 249 | // store, and the mark goes in the same write, so the seed's finish cannot |
| 250 | // tear down a store another host now uses. |
| 251 | func TestInstanceStoreProvisionAdoptingASeedsStoreClearsTheMark(t *testing.T) { |
| 252 | fx, fake, acct := seededStoreFixture(t, "code-seed-adopted") |
| 253 | fake.FailNext = map[string]error{"EnsureUser": errors.New("iam is down")} |
| 254 | w := httptest.NewRecorder() |
| 255 | if _, ok, _ := fx.srv.instanceStoreProvisionFor(context.Background(), w, acct, "shop", "canary", "alice@keymail.dev", seedA); ok { |
| 256 | t.Fatal("the seeded provision succeeded, want its walk to fail after the grant block") |
| 257 | } |
| 258 | if rec := canaryStore(t, fx); rec == nil || rec.Grant == nil || rec.Declared.Seed != seedA { |
| 259 | t.Fatalf("canary record = %+v, want the seed's grant block and mark", rec) |
| 260 | } |
| 261 | w = httptest.NewRecorder() |
| 262 | out, ok := fx.srv.instanceStoreProvision(context.Background(), w, acct, "shop", "canary", "bob@keymail.dev") |
| 263 | if !ok { |
| 264 | t.Fatalf("ordinary provision refused: %d %s", w.Code, w.Body.String()) |
| 265 | } |
| 266 | for _, rec := range []*objstore.Record{out, canaryStore(t, fx)} { |
| 267 | if rec.Declared == nil || rec.Declared.Seed != "" || rec.Provisioned == nil { |
| 268 | t.Fatalf("store after adoption = %+v / %+v, want provisioned with no seed mark", rec, rec.Declared) |
| 269 | } |
| 270 | } |
| 271 | } |
| 272 | |
| 273 | // Without a seed nothing is rolled back: a failed grant leaves the |
| 274 | // declaration standing, as it always has, for a retry to resume. |
| 275 | func TestInstanceStoreProvisionLeavesItsDeclarationOnAFailedGrant(t *testing.T) { |
| 276 | fx, _, acct := seededStoreFixture(t, "code-unseeded-failed-grant") |
| 277 | takeCanaryBucketName(t, fx) |
| 278 | w := httptest.NewRecorder() |
| 279 | if _, ok := fx.srv.instanceStoreProvision(context.Background(), w, acct, "shop", "canary", "alice@keymail.dev"); ok || w.Code != http.StatusConflict { |
| 280 | t.Fatalf("provision = %v (%d %s); want the name-taken refusal", ok, w.Code, w.Body.String()) |
| 281 | } |
| 282 | rec := canaryStore(t, fx) |
| 283 | if rec == nil || rec.Declared == nil || rec.Declared.Seed != "" || rec.Declared.By != "alice@keymail.dev" || rec.Grant != nil { |
| 284 | t.Fatalf("canary record = %+v, want the unmarked declaration left standing", rec) |
| 285 | } |
| 286 | } |
| 287 | |
| 288 | // Without a seed, a declared-only record keeps its author and timestamp, |
| 289 | // exactly as before: only a seeded create re-declares. A seed's mark on it |
| 290 | // does not survive the grant: the store is this create's, not the seed's, |
| 291 | // and the seed's finish must never tear it down. |
| 292 | func TestInstanceStoreProvisionKeepsAnExistingDeclaration(t *testing.T) { |
| 293 | fx, _, acct := seededStoreFixture(t, "code-unseeded-keep") |
| 294 | at := time.Now().UTC().Add(-time.Hour).Truncate(time.Second) |
| 295 | putCanaryStore(t, fx, &objstore.Record{Declared: &objstore.Declared{EnvPrefix: "CARLOS_STORE", At: at, By: "alice@keymail.dev", Seed: seedB}}) |
| 296 | w := httptest.NewRecorder() |
| 297 | out, ok := fx.srv.instanceStoreProvision(context.Background(), w, acct, "shop", "canary", "bob@keymail.dev") |
| 298 | if !ok { |
| 299 | t.Fatalf("provision refused: %d %s", w.Code, w.Body.String()) |
| 300 | } |
| 301 | for _, rec := range []*objstore.Record{out, canaryStore(t, fx)} { |
| 302 | if rec.Declared == nil || rec.Declared.By != "alice@keymail.dev" || !rec.Declared.At.Equal(at) || rec.Declared.Seed != "" || rec.Grant == nil { |
| 303 | t.Fatalf("declaration = %+v, want the existing one with the seed's mark cleared by the grant", rec.Declared) |
| 304 | } |
| 305 | } |
| 306 | } |
| 307 | |
amadan