| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "encoding/json" |
| 6 | "log" |
| 7 | "os" |
| 8 | "strings" |
| 9 | "testing" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/domains" |
| 13 | "github.com/carlosframework/platform/internal/release" |
| 14 | ) |
| 15 | |
| 16 | // The bucket is the control plane, and the console is not the only minter: |
| 17 | // `carlos accounts create` writes accounts/<sqid>/account.json and a registry |
| 18 | // row without ever opening the console database. Before this, the console |
| 19 | // simply did not know such an account existed — so `console member -add` would |
| 20 | // refuse a sqid that demonstrably exists, on the very box that created it. |
| 21 | func TestMaterializeAccountsFromTheBucket(t *testing.T) { |
| 22 | st := openTestStore(t) |
| 23 | rel := release.NewDirStore(t.TempDir()) |
| 24 | ctx := t.Context() |
| 25 | created := time.Unix(1700000000, 0).UTC() |
| 26 | |
| 27 | const kassID, blogID = int64(793481920157), int64(42) |
| 28 | for _, a := range []struct { |
| 29 | id int64 |
| 30 | name string |
| 31 | }{{kassID, "Kass"}, {blogID, "Blog"}} { |
| 32 | if err := WriteAccountManifest(ctx, rel, a.id, domains.Sqid(a.id), a.name, created); err != nil { |
| 33 | t.Fatal(err) |
| 34 | } |
| 35 | } |
| 36 | |
| 37 | n, err := MaterializeAccounts(ctx, rel, st) |
| 38 | if err != nil { |
| 39 | t.Fatal(err) |
| 40 | } |
| 41 | if n != 2 { |
| 42 | t.Errorf("materialized %d accounts, want 2", n) |
| 43 | } |
| 44 | acct, err := st.AccountBySqid(domains.Sqid(kassID)) |
| 45 | if err != nil || acct == nil { |
| 46 | t.Fatalf("AccountBySqid(Kass) = %v %v, want the materialized account", acct, err) |
| 47 | } |
| 48 | // The id in the manifest is taken as GIVEN. Minting a new one here would |
| 49 | // re-home every platform domain the account owns. |
| 50 | if acct.ID != kassID { |
| 51 | t.Errorf("materialized id = %d, want the manifest's %d", acct.ID, kassID) |
| 52 | } |
| 53 | if acct.Name != "Kass" || !acct.CreatedAt.Equal(created) { |
| 54 | t.Errorf("materialized account = %+v, want Kass created %v", acct, created) |
| 55 | } |
| 56 | |
| 57 | // Idempotent: a second pass finds nothing new, which is what makes it safe |
| 58 | // to run on every console start and before every membership operation. |
| 59 | if n, err := MaterializeAccounts(ctx, rel, st); err != nil || n != 0 { |
| 60 | t.Errorf("second pass materialized %d (%v), want 0", n, err) |
| 61 | } |
| 62 | } |
| 63 | |
| 64 | // A manifest whose sqid disagrees with its id is corruption, not a new |
| 65 | // account: materializing it would put an account at a domain that stops |
| 66 | // resolving the moment anything recomputes the sqid. Skipped loudly, never |
| 67 | // guessed at — the same rule internal/domains applies to the same objects. |
| 68 | func TestMaterializeSkipsManifestsThatDisagreeWithThemselves(t *testing.T) { |
| 69 | st := openTestStore(t) |
| 70 | rel := release.NewDirStore(t.TempDir()) |
| 71 | ctx := t.Context() |
| 72 | |
| 73 | // sqid in the path and body agree with each other but not with Sqid(id). |
| 74 | if _, err := rel.Put(ctx, "accounts/wrong/account.json", |
| 75 | []byte(`{"id":42,"sqid":"wrong","name":"Impostor"}`), ""); err != nil { |
| 76 | t.Fatal(err) |
| 77 | } |
| 78 | // Body sqid disagrees with the path. |
| 79 | if _, err := rel.Put(ctx, "accounts/"+domains.Sqid(43)+"/account.json", |
| 80 | []byte(`{"id":43,"sqid":"elsewhere","name":"Mislabelled"}`), ""); err != nil { |
| 81 | t.Fatal(err) |
| 82 | } |
| 83 | if _, err := rel.Put(ctx, "accounts/broken/account.json", []byte("{not json"), ""); err != nil { |
| 84 | t.Fatal(err) |
| 85 | } |
| 86 | if _, err := rel.Put(ctx, "accounts/empty/account.json", []byte("{}"), ""); err != nil { |
| 87 | t.Fatal(err) |
| 88 | } |
| 89 | // One good manifest alongside: a bad neighbour must not stop the pass. |
| 90 | if err := WriteAccountManifest(ctx, rel, 42, domains.Sqid(42), "Blog", time.Now()); err != nil { |
| 91 | t.Fatal(err) |
| 92 | } |
| 93 | |
| 94 | n, err := MaterializeAccounts(ctx, rel, st) |
| 95 | if err != nil { |
| 96 | t.Fatal(err) |
| 97 | } |
| 98 | if n != 1 { |
| 99 | t.Errorf("materialized %d, want only the well-formed manifest", n) |
| 100 | } |
| 101 | for _, sqid := range []string{"wrong", "elsewhere", "broken", "empty", domains.Sqid(43)} { |
| 102 | if a, _ := st.AccountBySqid(sqid); a != nil { |
| 103 | t.Errorf("materialized %q from a manifest that disagrees with itself: %+v", sqid, a) |
| 104 | } |
| 105 | } |
| 106 | if a, _ := st.AccountBySqid(domains.Sqid(42)); a == nil { |
| 107 | t.Error("the well-formed manifest was not materialized") |
| 108 | } |
| 109 | } |
| 110 | |
| 111 | // Materialization is INSERT-only. An account the console already knows keeps |
| 112 | // its row: the console is where names are edited, the manifest is written |
| 113 | // once, and letting a stale manifest name win would rename accounts behind |
| 114 | // the operator's back. Above all it must never reassign an id. |
| 115 | func TestMaterializeNeverRewritesAKnownAccount(t *testing.T) { |
| 116 | st := openTestStore(t) |
| 117 | rel := release.NewDirStore(t.TempDir()) |
| 118 | ctx := t.Context() |
| 119 | now := time.Unix(1700000000, 0).UTC() |
| 120 | |
| 121 | if err := st.CreateAccount(42, domains.Sqid(42), "Current name", "paul@keymail.dev", now); err != nil { |
| 122 | t.Fatal(err) |
| 123 | } |
| 124 | if err := WriteAccountManifest(ctx, rel, 42, domains.Sqid(42), "Stale name", now); err != nil { |
| 125 | t.Fatal(err) |
| 126 | } |
| 127 | if n, err := MaterializeAccounts(ctx, rel, st); err != nil || n != 0 { |
| 128 | t.Fatalf("materialized %d (%v) for an account the console already had", n, err) |
| 129 | } |
| 130 | acct, _ := st.AccountBySqid(domains.Sqid(42)) |
| 131 | if acct == nil || acct.Name != "Current name" || acct.ID != 42 { |
| 132 | t.Errorf("known account was rewritten: %+v", acct) |
| 133 | } |
| 134 | // And its membership survives — materialization touches memberships never. |
| 135 | if role, _ := st.Role(42, "paul@keymail.dev"); role != RoleOwner { |
| 136 | t.Errorf("owner role after materialize = %q, want owner", role) |
| 137 | } |
| 138 | } |
| 139 | |
| 140 | // FOUND IN PRODUCTION: `carlos accounts create` recorded an owner NOWHERE, so |
| 141 | // the accounts kass and keymail were migrated into had no members — and |
| 142 | // membership IS authorization, so the operator could not reach either of them |
| 143 | // in the console at all. The account existed, the apps were in it, and the only |
| 144 | // door was a `console member -add … -role owner` that nothing told anybody to |
| 145 | // run. |
| 146 | // |
| 147 | // The fix keeps the boundary the carlos binary is held to (it must never open |
| 148 | // this database): `create` names the owner in the BUCKET manifest, and this |
| 149 | // pass — the console's own reader of that manifest — turns it into the owner |
| 150 | // membership. Same object, same reader, on every box. |
| 151 | func TestMaterializeGivesANewAccountTheManifestOwner(t *testing.T) { |
| 152 | st := openTestStore(t) |
| 153 | rel := release.NewDirStore(t.TempDir()) |
| 154 | ctx := t.Context() |
| 155 | |
| 156 | // Exactly what `carlos accounts create -name Kass -owner paul@keymail.dev` |
| 157 | // leaves behind, minted through the same allocator. |
| 158 | id, sqid, err := domains.MintAccount(ctx, rel, "Kass", "paul@keymail.dev", time.Unix(1700000000, 0)) |
| 159 | if err != nil { |
| 160 | t.Fatal(err) |
| 161 | } |
| 162 | if n, err := MaterializeAccounts(ctx, rel, st); err != nil || n != 1 { |
| 163 | t.Fatalf("materialized %d (%v), want 1", n, err) |
| 164 | } |
| 165 | if role, _ := st.Role(id, "paul@keymail.dev"); role != RoleOwner { |
| 166 | t.Errorf("owner role after materialize = %q, want owner — the account is unreachable without it", role) |
| 167 | } |
| 168 | // And it lands where the console looks it up from: the switcher. |
| 169 | accts, err := st.AccountsFor("paul@keymail.dev") |
| 170 | if err != nil { |
| 171 | t.Fatal(err) |
| 172 | } |
| 173 | if len(accts) != 1 || accts[0].Sqid != sqid || accts[0].Role != RoleOwner { |
| 174 | t.Errorf("AccountsFor(owner) = %+v, want the new account as owner", accts) |
| 175 | } |
| 176 | // Idempotent: the account is known and owned, so a second pass does nothing. |
| 177 | if n, err := MaterializeAccounts(ctx, rel, st); err != nil || n != 0 { |
| 178 | t.Errorf("second pass materialized %d (%v), want 0", n, err) |
| 179 | } |
| 180 | } |
| 181 | |
| 182 | // THE MANIFEST IS NOT WRITE-ONCE, and the owner is only adopted at BIRTH. |
| 183 | // |
| 184 | // `MintAccount` claims the object with a create-only Put, but that is a |
| 185 | // convention of one code path, not a property of the object: release.Store.Put |
| 186 | // takes its ifMatch from the caller, the deployment bucket has no Object Lock, |
| 187 | // and the host IAM role holds unconditional s3:PutObject over the whole bucket |
| 188 | // with nothing excluding accounts/*/account.json. So anyone who can write to |
| 189 | // the bucket can REWRITE a live account's manifest. |
| 190 | // |
| 191 | // If materialization adopted an owner on any pass, that rewrite would be an |
| 192 | // authority grant: name yourself owner of a vacant account, wait for the next |
| 193 | // console start or `console member` invocation — both run this pass — and you |
| 194 | // hold owner over a live account with its apps in it. That is a genuine |
| 195 | // widening of the trust model, not something bucket write access already |
| 196 | // implied: forging a channel pointer needs CARLOS_RELEASE_KEY, which most |
| 197 | // bucket-writers do not have, and this path needs no key at all. |
| 198 | // |
| 199 | // The transactional vacancy check in AdoptManifestOwner does not help here. It |
| 200 | // is TOCTOU-safe, but the account was LEGITIMATELY vacant; the manifest is what |
| 201 | // changed afterwards. |
| 202 | // |
| 203 | // So adoption happens once, when the console first creates the account row, and |
| 204 | // never again. Safe now that -owner is required at create time: manifests are |
| 205 | // born with their owner. |
| 206 | func TestMaterializeNeverAdoptsAnOwnerAddedToAManifestAfterTheFact(t *testing.T) { |
| 207 | st := openTestStore(t) |
| 208 | rel := release.NewDirStore(t.TempDir()) |
| 209 | ctx := t.Context() |
| 210 | now := time.Unix(1700000000, 0) |
| 211 | |
| 212 | // A legitimately vacant account: an old manifest, from before `owner`. |
| 213 | if err := WriteAccountManifest(ctx, rel, 42, domains.Sqid(42), "Kass", now); err != nil { |
| 214 | t.Fatal(err) |
| 215 | } |
| 216 | if n, err := MaterializeAccounts(ctx, rel, st); err != nil || n != 1 { |
| 217 | t.Fatalf("materialized %d (%v), want 1", n, err) |
| 218 | } |
| 219 | if members, _ := st.Members(42); len(members) != 0 { |
| 220 | t.Fatalf("Members = %+v, want the account vacant to begin with", members) |
| 221 | } |
| 222 | |
| 223 | // Somebody with plain bucket write access — no signing key — rewrites the |
| 224 | // manifest naming themselves. |
| 225 | rewriteManifest(t, rel, 42, "attacker@keymail.dev") |
| 226 | // This pass runs on every console start and before every `console member`. |
| 227 | if _, err := MaterializeAccounts(ctx, rel, st); err != nil { |
| 228 | t.Fatal(err) |
| 229 | } |
| 230 | if role, _ := st.Role(42, "attacker@keymail.dev"); role != "" { |
| 231 | t.Errorf("a rewritten manifest granted %q over a live account — bucket write became "+ |
| 232 | "console authority, with no signing key involved", role) |
| 233 | } |
| 234 | if members, _ := st.Members(42); len(members) != 0 { |
| 235 | t.Errorf("Members = %+v, want the account still vacant", members) |
| 236 | } |
| 237 | } |
| 238 | |
| 239 | // The same rule stated the other way, and the case an operator will actually |
| 240 | // hit: an owner added to a manifest AFTER the account first materialized is |
| 241 | // deliberately not adopted, however well-meant. The operator path for that is |
| 242 | // `console member -add <address> <sqid> -role owner`, which is box-side |
| 243 | // authority rather than bucket-side. |
| 244 | func TestMaterializeLeavesALateManifestOwnerToTheOperator(t *testing.T) { |
| 245 | st := openTestStore(t) |
| 246 | rel := release.NewDirStore(t.TempDir()) |
| 247 | ctx := t.Context() |
| 248 | now := time.Unix(1700000000, 0) |
| 249 | |
| 250 | if err := WriteAccountManifest(ctx, rel, 42, domains.Sqid(42), "Kass", now); err != nil { |
| 251 | t.Fatal(err) |
| 252 | } |
| 253 | if _, err := MaterializeAccounts(ctx, rel, st); err != nil { |
| 254 | t.Fatal(err) |
| 255 | } |
| 256 | rewriteManifest(t, rel, 42, "paul@keymail.dev") |
| 257 | if _, err := MaterializeAccounts(ctx, rel, st); err != nil { |
| 258 | t.Fatal(err) |
| 259 | } |
| 260 | if role, _ := st.Role(42, "paul@keymail.dev"); role != RoleOwner { |
| 261 | // The box-side verb is what fills it, and it still works. |
| 262 | if err := st.AddMember(42, "paul@keymail.dev", RoleOwner, now); err != nil { |
| 263 | t.Fatal(err) |
| 264 | } |
| 265 | } else { |
| 266 | t.Error("a late manifest owner was adopted — the manifest is not write-once, " + |
| 267 | "so that is a standing grant anyone with bucket write can issue") |
| 268 | } |
| 269 | } |
| 270 | |
| 271 | // THE COUPLING TEST. Once an owner exists, the only thing standing between "the |
| 272 | // operator removed them" and "the manifest puts them back" is that the account |
| 273 | // never becomes vacant again — and that is enforced by lastOwnerGuard, which |
| 274 | // lives in member removal, not here. This diff leans on that guard implicitly; |
| 275 | // exercising it end to end means a future change to removal (or to account |
| 276 | // deletion) cannot silently reopen the adoption path. |
| 277 | func TestARemovedOwnerIsNeverRegrantedByTheManifest(t *testing.T) { |
| 278 | st := openTestStore(t) |
| 279 | rel := release.NewDirStore(t.TempDir()) |
| 280 | ctx := t.Context() |
| 281 | now := time.Unix(1700000000, 0) |
| 282 | |
| 283 | id, _, err := domains.MintAccount(ctx, rel, "Kass", "paul@keymail.dev", now) |
| 284 | if err != nil { |
| 285 | t.Fatal(err) |
| 286 | } |
| 287 | if _, err := MaterializeAccounts(ctx, rel, st); err != nil { |
| 288 | t.Fatal(err) |
| 289 | } |
| 290 | if role, _ := st.Role(id, "paul@keymail.dev"); role != RoleOwner { |
| 291 | t.Fatalf("role = %q, want the manifest owner adopted at birth", role) |
| 292 | } |
| 293 | |
| 294 | // Link one: the account cannot be emptied. Removing the last owner is |
| 295 | // refused, so there is no vacancy for a re-read to fill. |
| 296 | if err := st.RemoveMember(id, "paul@keymail.dev"); err == nil { |
| 297 | t.Fatal("removed the last owner — the account is now vacant and the manifest can refill it") |
| 298 | } |
| 299 | |
| 300 | // Link two: with ownership transferred, the original CAN be removed — and |
| 301 | // must stay removed across every subsequent pass, forever. |
| 302 | if err := st.AddMember(id, "successor@keymail.dev", RoleOwner, now); err != nil { |
| 303 | t.Fatal(err) |
| 304 | } |
| 305 | if err := st.RemoveMember(id, "paul@keymail.dev"); err != nil { |
| 306 | t.Fatal(err) |
| 307 | } |
| 308 | for i := 0; i < 3; i++ { |
| 309 | if _, err := MaterializeAccounts(ctx, rel, st); err != nil { |
| 310 | t.Fatal(err) |
| 311 | } |
| 312 | } |
| 313 | if role, _ := st.Role(id, "paul@keymail.dev"); role != "" { |
| 314 | t.Errorf("the removed owner came back as %q — the manifest is behaving as a standing grant", role) |
| 315 | } |
| 316 | if role, _ := st.Role(id, "successor@keymail.dev"); role != RoleOwner { |
| 317 | t.Errorf("successor role = %q, want owner", role) |
| 318 | } |
| 319 | } |
| 320 | |
| 321 | // The manifest is written ONCE and read on every start, so it must never be a |
| 322 | // standing grant. An account that has an owner is administered by the console; |
| 323 | // re-reading the object must not demote them, replace them, or add the |
| 324 | // manifest's address alongside them. |
| 325 | func TestMaterializeNeverReplacesAnExistingOwner(t *testing.T) { |
| 326 | st := openTestStore(t) |
| 327 | rel := release.NewDirStore(t.TempDir()) |
| 328 | ctx := t.Context() |
| 329 | now := time.Unix(1700000000, 0) |
| 330 | |
| 331 | id, sqid, err := domains.MintAccount(ctx, rel, "Kass", "original@keymail.dev", now) |
| 332 | if err != nil { |
| 333 | t.Fatal(err) |
| 334 | } |
| 335 | // Ownership was transferred in the console after creation. |
| 336 | if err := st.CreateAccount(id, sqid, "Kass", "successor@keymail.dev", now); err != nil { |
| 337 | t.Fatal(err) |
| 338 | } |
| 339 | if _, err := MaterializeAccounts(ctx, rel, st); err != nil { |
| 340 | t.Fatal(err) |
| 341 | } |
| 342 | if role, _ := st.Role(id, "successor@keymail.dev"); role != RoleOwner { |
| 343 | t.Errorf("the console's owner became %q after a materialize pass", role) |
| 344 | } |
| 345 | if role, _ := st.Role(id, "original@keymail.dev"); role != "" { |
| 346 | t.Errorf("the manifest re-granted %q to the address it named at creation — "+ |
| 347 | "a write-once object must not be a standing grant", role) |
| 348 | } |
| 349 | } |
| 350 | |
| 351 | // An owner address the sign-in form would refuse is a membership nobody can |
| 352 | // ever claim. Skipped loudly, and the ACCOUNT still materializes: the identity |
| 353 | // is sound even when the owner field is not, and refusing the whole manifest |
| 354 | // would make one bad address hide an account from the console entirely. |
| 355 | func TestMaterializeSkipsAnUnusableManifestOwner(t *testing.T) { |
| 356 | st := openTestStore(t) |
| 357 | rel := release.NewDirStore(t.TempDir()) |
| 358 | ctx := t.Context() |
| 359 | |
| 360 | sqid := domains.Sqid(42) |
| 361 | if _, err := rel.Put(ctx, domains.AccountManifestKey(sqid), |
| 362 | []byte(`{"id":42,"sqid":"`+sqid+`","name":"Kass","owner":"not-an-address"}`), ""); err != nil { |
| 363 | t.Fatal(err) |
| 364 | } |
| 365 | if n, err := MaterializeAccounts(ctx, rel, st); err != nil || n != 1 { |
| 366 | t.Fatalf("materialized %d (%v), want the account itself", n, err) |
| 367 | } |
| 368 | if members, _ := st.Members(42); len(members) != 0 { |
| 369 | t.Errorf("Members = %+v, want nobody — an address keymail cannot authenticate is not an owner", members) |
| 370 | } |
| 371 | } |
| 372 | |
| 373 | // An old manifest — written before the field existed — has no owner and stays |
| 374 | // ownerless. Additive means the pre-owner world is unchanged. |
| 375 | func TestMaterializeLeavesAnOwnerlessManifestOwnerless(t *testing.T) { |
| 376 | st := openTestStore(t) |
| 377 | rel := release.NewDirStore(t.TempDir()) |
| 378 | ctx := t.Context() |
| 379 | if err := WriteAccountManifest(ctx, rel, 42, domains.Sqid(42), "Legacy", time.Unix(1700000000, 0)); err != nil { |
| 380 | t.Fatal(err) |
| 381 | } |
| 382 | if n, err := MaterializeAccounts(ctx, rel, st); err != nil || n != 1 { |
| 383 | t.Fatalf("materialized %d (%v), want 1", n, err) |
| 384 | } |
| 385 | if members, _ := st.Members(42); len(members) != 0 { |
| 386 | t.Errorf("Members = %+v, want nobody — inventing an owner is minting authority from nothing", members) |
| 387 | } |
| 388 | } |
| 389 | |
| 390 | // THE KASS GAP, closed: a deletion performed anywhere becomes real here. |
| 391 | // |
| 392 | // Before this, MaterializeAccounts merely SKIPPED a tombstoned manifest, so an |
| 393 | // account deleted on another box (or by `carlos accounts delete`) kept its |
| 394 | // console row, its memberships and its apps forever — the console said the |
| 395 | // account was live, the control plane said it was gone, and only hand-run SQL |
| 396 | // could settle it. The bucket is the control plane: it has already ruled. |
| 397 | func TestMaterializeReapsTombstonedAccounts(t *testing.T) { |
| 398 | st := openTestStore(t) |
| 399 | rel := release.NewDirStore(t.TempDir()) |
| 400 | ctx := t.Context() |
| 401 | now := time.Unix(1700000000, 0).UTC() |
| 402 | |
| 403 | id, sqid, err := domains.MintAccount(ctx, rel, "Testco", "paul@keymail.dev", now) |
| 404 | if err != nil { |
| 405 | t.Fatal(err) |
| 406 | } |
| 407 | if n, err := MaterializeAccounts(ctx, rel, st); err != nil || n != 1 { |
| 408 | t.Fatalf("materialized %d (%v), want the new account", n, err) |
| 409 | } |
| 410 | if _, err := st.CreateAppIn(id, "blog", "paul@keymail.dev", now); err != nil { |
| 411 | t.Fatal(err) |
| 412 | } |
| 413 | if err := st.SetLastAccount("paul@keymail.dev", id); err != nil { |
| 414 | t.Fatal(err) |
| 415 | } |
| 416 | |
| 417 | // Deleted somewhere else entirely, while this console knew nothing about it. |
| 418 | if err := domains.TombstoneAccount(ctx, rel, sqid, now); err != nil { |
| 419 | t.Fatal(err) |
| 420 | } |
| 421 | if n, err := MaterializeAccounts(ctx, rel, st); err != nil || n != 0 { |
| 422 | t.Fatalf("materialized %d (%v) on the reaping pass, want 0 — a reap is not a birth", n, err) |
| 423 | } |
| 424 | |
| 425 | if a, err := st.AccountBySqid(sqid); err != nil || a != nil { |
| 426 | t.Errorf("AccountBySqid after the tombstone = %+v %v, want nothing: the control plane has ruled", a, err) |
| 427 | } |
| 428 | // Invisible, but REMEMBERED — see the resurrection test for what forgetting costs. |
| 429 | if a, deleted, err := st.AccountAnyBySqid(sqid); err != nil || a == nil || !deleted { |
| 430 | t.Fatalf("AccountAnyBySqid = %+v %v %v, want a marked row", a, deleted, err) |
| 431 | } |
| 432 | if members, _ := st.Members(id); len(members) != 0 { |
| 433 | t.Errorf("Members = %+v, want none — membership against a dead account is access to nothing", members) |
| 434 | } |
| 435 | if apps, _ := st.AppsIn(id); len(apps) != 0 { |
| 436 | t.Errorf("AppsIn = %v, want none", apps) |
| 437 | } |
| 438 | if trashed, _ := st.TrashedAppsIn(id); len(trashed) != 0 { |
| 439 | t.Errorf("TrashedAppsIn = %+v, want none — the app rows are reaped with the account", trashed) |
| 440 | } |
| 441 | if last, err := st.LastAccount("paul@keymail.dev"); err != nil || last != 0 { |
| 442 | t.Errorf("LastAccount = %d %v, want 0 — the MRU must not land anyone in a reaped account", last, err) |
| 443 | } |
| 444 | // Stable: a second pass finds a marked row and a tombstoned manifest agreeing. |
| 445 | if n, err := MaterializeAccounts(ctx, rel, st); err != nil || n != 0 { |
| 446 | t.Errorf("second pass materialized %d (%v), want 0", n, err) |
| 447 | } |
| 448 | } |
| 449 | |
| 450 | // THE SECURITY TEST. Owner adoption is birth-only, and "birth" must mean |
| 451 | // FIRST-EVER — not "first since the row went away". |
| 452 | // |
| 453 | // Deleting the row would forget the account was ever seen, and forgetting |
| 454 | // re-arms the whole attack materializeOwner exists to prevent: tombstone an |
| 455 | // account (or wait for someone to delete one), let the console reap it, then |
| 456 | // rewrite the manifest un-tombstoned with yourself as owner. A console that |
| 457 | // treats that as a first materialization re-births the account — same id, same |
| 458 | // sqid, same platform hostnames, same apps prefix — and adopts the attacker. |
| 459 | // Bucket write access, no signing key, a whole account. |
| 460 | // |
| 461 | // So a marked row is tombstone MEMORY, and memory does not argue with a |
| 462 | // rewritten manifest. An account id is spent once. |
| 463 | func TestMaterializeRefusesResurrection(t *testing.T) { |
| 464 | st := openTestStore(t) |
| 465 | rel := release.NewDirStore(t.TempDir()) |
| 466 | ctx := t.Context() |
| 467 | now := time.Unix(1700000000, 0).UTC() |
| 468 | |
| 469 | id, sqid, err := domains.MintAccount(ctx, rel, "Testco", "paul@keymail.dev", now) |
| 470 | if err != nil { |
| 471 | t.Fatal(err) |
| 472 | } |
| 473 | if n, err := MaterializeAccounts(ctx, rel, st); err != nil || n != 1 { |
| 474 | t.Fatalf("materialized %d (%v), want the new account", n, err) |
| 475 | } |
| 476 | if err := domains.TombstoneAccount(ctx, rel, sqid, now); err != nil { |
| 477 | t.Fatal(err) |
| 478 | } |
| 479 | if _, err := MaterializeAccounts(ctx, rel, st); err != nil { |
| 480 | t.Fatal(err) |
| 481 | } |
| 482 | if _, deleted, _ := st.AccountAnyBySqid(sqid); !deleted { |
| 483 | t.Fatal("the account was not reaped, so this test is not testing what it says") |
| 484 | } |
| 485 | |
| 486 | // Somebody with plain s3:PutObject — no CARLOS_RELEASE_KEY, no console |
| 487 | // session — writes the manifest back, live, naming themselves. |
| 488 | rewriteManifest(t, rel, id, "eve@evil.example") |
| 489 | |
| 490 | // The refusal must be EXPLICIT, not an accident of some later check. Delete |
| 491 | // the corruption arm and every assertion below still passes — refusal falls |
| 492 | // through to "known account, stop here" — while the operator loses the only |
| 493 | // signal that somebody rewrote a dead account's manifest. So the log line is |
| 494 | // part of the contract and is pinned here. |
| 495 | var logged bytes.Buffer |
| 496 | log.SetOutput(&logged) |
| 497 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 498 | |
| 499 | // Every console start runs this pass, and so does every `console member`. |
| 500 | for i := 0; i < 2; i++ { |
| 501 | if n, err := MaterializeAccounts(ctx, rel, st); err != nil || n != 0 { |
| 502 | t.Fatalf("pass %d materialized %d (%v) — a spent account id was re-birthed from a rewritten manifest", i, n, err) |
| 503 | } |
| 504 | } |
| 505 | if a, _ := st.AccountBySqid(sqid); a != nil { |
| 506 | t.Errorf("the account came back as %+v — reap, rewrite, re-birth is an account takeover", a) |
| 507 | } |
| 508 | if _, deleted, _ := st.AccountAnyBySqid(sqid); !deleted { |
| 509 | t.Error("the marked row lost its mark — the console has forgotten the tombstone") |
| 510 | } |
| 511 | if role, _ := st.Role(id, "eve@evil.example"); role != "" { |
| 512 | t.Errorf("the rewritten manifest granted %q — bucket write became console authority", role) |
| 513 | } |
| 514 | if accts, _ := st.AccountsFor("eve@evil.example"); len(accts) != 0 { |
| 515 | t.Errorf("AccountsFor(attacker) = %+v, want nothing adopted", accts) |
| 516 | } |
| 517 | if members, _ := st.Members(id); len(members) != 0 { |
| 518 | t.Errorf("Members = %+v, want none", members) |
| 519 | } |
| 520 | if !strings.Contains(logged.String(), "refusing to resurrect") { |
| 521 | t.Errorf("nothing said a word about the rewritten manifest; log was:\n%s\n"+ |
| 522 | "a manifest that came back to life over a tombstoned account is corruption, "+ |
| 523 | "and an operator has to hear about it", logged.String()) |
| 524 | } |
| 525 | } |
| 526 | |
| 527 | // App-row convergence: the console's rows follow the bucket, because trash and |
| 528 | // purge can both happen while this console is down (headless `carlos`, another |
| 529 | // box, the auto-purge pass). Four cases, and case (d) is the one that keeps |
| 530 | // convergence honest. |
| 531 | func TestMaterializeAppsConverges(t *testing.T) { |
| 532 | st := openTestStore(t) |
| 533 | rel := release.NewDirStore(t.TempDir()) |
| 534 | ctx := t.Context() |
| 535 | now := time.Unix(1700000000, 0).UTC() |
| 536 | trashedAt := now.Add(-48 * time.Hour) |
| 537 | |
| 538 | id, sqid, err := domains.MintAccount(ctx, rel, "Testco", "paul@keymail.dev", now) |
| 539 | if err != nil { |
| 540 | t.Fatal(err) |
| 541 | } |
| 542 | if n, err := MaterializeAccounts(ctx, rel, st); err != nil || n != 1 { |
| 543 | t.Fatalf("materialized %d (%v), want the account", n, err) |
| 544 | } |
| 545 | for _, app := range []string{"alpha", "beta", "gamma", "delta"} { |
| 546 | if _, err := st.CreateAppIn(id, app, "paul@keymail.dev", now); err != nil { |
| 547 | t.Fatal(err) |
| 548 | } |
| 549 | } |
| 550 | shipped := func(app string) { |
| 551 | t.Helper() |
| 552 | if _, err := rel.Put(ctx, "apps/"+sqid+"/"+app+"/channels/stable.json", []byte(`{}`), ""); err != nil { |
| 553 | t.Fatal(err) |
| 554 | } |
| 555 | } |
| 556 | // (a) live row, trashed headlessly: the marker exists in the bucket. |
| 557 | shipped("alpha") |
| 558 | if err := release.TrashApp(ctx, rel, sqid, "alpha", "paul@keymail.dev", trashedAt); err != nil { |
| 559 | t.Fatal(err) |
| 560 | } |
| 561 | // (b) marked row, restored headlessly: marker gone, objects still there. |
| 562 | shipped("beta") |
| 563 | if err := st.TrashAppRow(id, "beta", trashedAt); err != nil { |
| 564 | t.Fatal(err) |
| 565 | } |
| 566 | // (c) marked row, purged: no marker and no objects at all. |
| 567 | if err := st.TrashAppRow(id, "gamma", trashedAt); err != nil { |
| 568 | t.Fatal(err) |
| 569 | } |
| 570 | // (d) live row, never shipped: no marker, no objects — a legitimate claim. |
| 571 | // Nothing about it is distinguishable from (c) except that its row is LIVE. |
| 572 | |
| 573 | // Another account's keys, for a sqid this console has never heard of: they |
| 574 | // must contribute nothing to anybody's decision. |
| 575 | if _, err := rel.Put(ctx, "apps/"+domains.Sqid(999999)+"/alpha/channels/stable.json", []byte(`{}`), ""); err != nil { |
| 576 | t.Fatal(err) |
| 577 | } |
| 578 | |
| 579 | if err := MaterializeApps(ctx, rel, st); err != nil { |
| 580 | t.Fatal(err) |
| 581 | } |
| 582 | |
| 583 | if live, _ := st.HasAppIn(id, "alpha"); live { |
| 584 | t.Error("(a) alpha has a trash marker in the bucket but its row is still live") |
| 585 | } |
| 586 | if trashed, _ := st.HasTrashedAppIn(id, "alpha"); !trashed { |
| 587 | t.Error("(a) alpha was not marked trashed from its bucket marker") |
| 588 | } |
| 589 | if live, _ := st.HasAppIn(id, "beta"); !live { |
| 590 | t.Error("(b) beta's marker is gone and its objects remain — the row should be live again") |
| 591 | } |
| 592 | if trashed, _ := st.HasTrashedAppIn(id, "gamma"); trashed { |
| 593 | t.Error("(c) gamma is purged in the bucket but its row still holds the name") |
| 594 | } |
| 595 | if live, _ := st.HasAppIn(id, "gamma"); live { |
| 596 | t.Error("(c) gamma came back to life") |
| 597 | } |
| 598 | if accts, _ := st.AppAccounts("gamma"); len(accts) != 0 { |
| 599 | t.Errorf("(c) gamma still has rows %v — the row IS the name reservation, so purge must free it", accts) |
| 600 | } |
| 601 | if live, _ := st.HasAppIn(id, "delta"); !live { |
| 602 | t.Error("(d) a claimed-but-never-shipped app was eaten by convergence — " + |
| 603 | "an app with no objects is indistinguishable from a purged one except by its own row") |
| 604 | } |
| 605 | |
| 606 | // The cooloff clock comes from the MARKER, not from this pass: the bucket |
| 607 | // records when the app was trashed, and a console catching up days later |
| 608 | // must not restart the 30 days. |
| 609 | trashedRows, err := st.TrashedAppsIn(id) |
| 610 | if err != nil { |
| 611 | t.Fatal(err) |
| 612 | } |
| 613 | if len(trashedRows) != 1 || trashedRows[0].App != "alpha" || !trashedRows[0].DeletedAt.Equal(trashedAt) { |
| 614 | t.Errorf("TrashedAppsIn = %+v, want alpha trashed at %v", trashedRows, trashedAt) |
| 615 | } |
| 616 | |
| 617 | // Idempotent: convergence has converged. |
| 618 | if err := MaterializeApps(ctx, rel, st); err != nil { |
| 619 | t.Fatal(err) |
| 620 | } |
| 621 | apps, _ := st.AppsIn(id) |
| 622 | if len(apps) != 2 || apps[0] != "beta" || apps[1] != "delta" { |
| 623 | t.Errorf("AppsIn after a second pass = %v, want [beta delta]", apps) |
| 624 | } |
| 625 | } |
| 626 | |
| 627 | // A MISTYPED BUCKET NAME MUST NOT FREE NAMES. |
| 628 | // |
| 629 | // Case (c) is the only branch that reasons from absence, and an empty listing |
| 630 | // is exactly what CARLOS_DEPLOYMENT_BUCKET pointing one character wrong |
| 631 | // produces: a listing that SUCCEEDS and returns nothing. Purging on that reads |
| 632 | // "everything was purged everywhere" from what is really "I am looking in the |
| 633 | // wrong place" — and it is unrecoverable, because this pass is row-driven: once |
| 634 | // the rows are gone, the markers in the real bucket are invisible to it forever |
| 635 | // and the names are free for anyone to claim. |
| 636 | func TestMaterializeAppsRefusesAnEmptyBucket(t *testing.T) { |
| 637 | st := openTestStore(t) |
| 638 | ctx := t.Context() |
| 639 | now := time.Unix(1700000000, 0).UTC() |
| 640 | |
| 641 | empty := release.NewDirStore(t.TempDir()) |
| 642 | id, sqid, err := domains.MintAccount(ctx, empty, "Testco", "paul@keymail.dev", now) |
| 643 | if err != nil { |
| 644 | t.Fatal(err) |
| 645 | } |
| 646 | if n, err := MaterializeAccounts(ctx, empty, st); err != nil || n != 1 { |
| 647 | t.Fatalf("materialized %d (%v), want the account", n, err) |
| 648 | } |
| 649 | if _, err := st.CreateAppIn(id, "gamma", "paul@keymail.dev", now); err != nil { |
| 650 | t.Fatal(err) |
| 651 | } |
| 652 | if err := st.TrashAppRow(id, "gamma", now.Add(-48*time.Hour)); err != nil { |
| 653 | t.Fatal(err) |
| 654 | } |
| 655 | |
| 656 | // The wrong bucket: account manifests happen to be there, nothing under |
| 657 | // apps/ is. (A genuinely wrong bucket has neither; this store is the |
| 658 | // STRONGER test, since it still lists something.) |
| 659 | var logged bytes.Buffer |
| 660 | log.SetOutput(&logged) |
| 661 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 662 | if err := MaterializeApps(ctx, empty, st); err != nil { |
| 663 | t.Fatal(err) |
| 664 | } |
| 665 | if trashed, _ := st.HasTrashedAppIn(id, "gamma"); !trashed { |
| 666 | t.Error("an empty apps/ listing released a held name — one wrong environment variable " + |
| 667 | "is now enough to free every trashed app name in the deployment") |
| 668 | } |
| 669 | if accts, _ := st.AppAccounts("gamma"); len(accts) != 1 { |
| 670 | t.Errorf("AppAccounts(gamma) = %v, want the reservation still held", accts) |
| 671 | } |
| 672 | if !strings.Contains(logged.String(), "CARLOS_DEPLOYMENT_BUCKET") { |
| 673 | t.Errorf("the refusal was silent; log was:\n%s\nan operator has to be told why "+ |
| 674 | "names are not being released", logged.String()) |
| 675 | } |
| 676 | |
| 677 | // And the tripwire is about EMPTINESS, not about this app: one unrelated |
| 678 | // key under apps/ is enough evidence that the bucket is the right bucket, |
| 679 | // and gamma — no marker, no keys of its own — purges as before. |
| 680 | if _, err := empty.Put(ctx, "apps/"+sqid+"/somethingelse/channels/stable.json", []byte(`{}`), ""); err != nil { |
| 681 | t.Fatal(err) |
| 682 | } |
| 683 | if err := MaterializeApps(ctx, empty, st); err != nil { |
| 684 | t.Fatal(err) |
| 685 | } |
| 686 | if accts, _ := st.AppAccounts("gamma"); len(accts) != 0 { |
| 687 | t.Errorf("AppAccounts(gamma) = %v after a non-empty listing, want the name freed — "+ |
| 688 | "the tripwire must hold back purges only while there is no evidence at all", accts) |
| 689 | } |
| 690 | } |
| 691 | |
| 692 | // rewriteManifest overwrites a live account manifest with one naming owner — |
| 693 | // what a bucket-writer does, and the thing MintAccount's create-only Put does |
| 694 | // NOT prevent anyone else from doing. The etag is read and passed because the |
| 695 | // DirStore emulates compare-and-swap; S3 has no such constraint at all, so a |
| 696 | // caller holding s3:PutObject simply writes the object. |
| 697 | func rewriteManifest(t *testing.T, rel release.Store, id int64, owner string) { |
| 698 | t.Helper() |
| 699 | ctx := t.Context() |
| 700 | sqid := domains.Sqid(id) |
| 701 | key := domains.AccountManifestKey(sqid) |
| 702 | _, etag, err := rel.Get(ctx, key) |
| 703 | if err != nil { |
| 704 | t.Fatal(err) |
| 705 | } |
| 706 | body, err := json.Marshal(domains.AccountManifest{ |
| 707 | ID: id, Sqid: sqid, Name: "Kass", Owner: owner, CreatedAt: time.Unix(1700000000, 0).UTC(), |
| 708 | }) |
| 709 | if err != nil { |
| 710 | t.Fatal(err) |
| 711 | } |
| 712 | if _, err := rel.Put(ctx, key, body, etag); err != nil { |
| 713 | t.Fatal(err) |
| 714 | } |
| 715 | } |
| 716 | |
| 717 | // A deployment with no bucket configured at all still has to work: this is |
| 718 | // best-effort catch-up, not a precondition. |
| 719 | func TestMaterializeWithNoStoreIsANoOp(t *testing.T) { |
| 720 | st := openTestStore(t) |
| 721 | if n, err := MaterializeAccounts(t.Context(), nil, st); err != nil || n != 0 { |
| 722 | t.Errorf("MaterializeAccounts(nil store) = %d %v, want 0 nil", n, err) |
| 723 | } |
| 724 | // And app convergence above all: with no bucket to compare against, "no |
| 725 | // keys" would otherwise read as "purged everywhere". |
| 726 | if err := MaterializeApps(t.Context(), nil, st); err != nil { |
| 727 | t.Errorf("MaterializeApps(nil store) = %v, want nil", err) |
| 728 | } |
| 729 | } |
| 730 | |
amadan