amadan
console: materialize accounts from the bucket — the console is not the only minter
`carlos accounts create` writes accounts/<sqid>/account.json and the registry row and never opens console state (deliberately: the carlos binary has no business inside the console's database). The console therefore never learned of those accounts, so `console member -add … <sqid>` refused a sqid that demonstrably exists — on the very box that had just created it. MaterializeAccounts reads the SAME manifests the domains reconciler reads, through the same key parser (domains.AccountManifestSqid, exported for it: two parsers could disagree about what counts as a manifest, and the account set is what tells a qualified bucket key from a legacy one). Runs on console startup and before every membership operation. The manifest's id is taken as GIVEN — nothing here mints or reassigns one. A manifest that disagrees with itself (path sqid, body sqid and Sqid(id) must be the same string) is corruption, not a new account, and is skipped loudly. INSERT-only, so a known account keeps its row, name and members. Store.EnsureAccount deliberately creates NO membership, unlike CreateAccount: `carlos accounts create` records no owner anywhere, so the console has nobody to name, and inventing one would mint authority out of thin air. An ownerless row is honest and harmless — `console member -add <addr> <sqid> -role owner` is how it gets an owner, which is now a tool that exists. Best-effort throughout: no bucket configured, or an unreachable one, must never be what stops an operator removing a membership.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
8 files changed,
+355
−6
cmd/console/main.go+13 −0cmd/console/member.go+13 −2cmd/console/member_test.go+42 −1internal/console/materialize.go+104 −0internal/console/materialize_test.go+142 −0internal/console/server.go+8 −0internal/console/store.go+24 −0internal/domains/reconcile.go+9 −3
diff --git a/cmd/console/main.go b/cmd/console/main.go| index bcd5db3d3..3f771b83c 100644 |
| --- a/cmd/console/main.go |
| +++ b/cmd/console/main.go |
| @@ -231,6 +231,19 @@ func openConsoleDB() *console.Store { |
| return st |
| } |
| +// optionalStore is openStore for the verbs that would LIKE the deployment |
| +// bucket but must work without it. `console member` reads account manifests |
| +// out of it to catch up on accounts minted by `carlos accounts create`; a box |
| +// with neither variable set is a legal deployment (grant/revoke/owners have |
| +// always needed only CONSOLE_DB), and refusing to run there would take away |
| +// the membership tooling on exactly the boxes least able to spare it. |
| +func optionalStore() release.Store { |
| + if os.Getenv("CARLOS_DEPLOYMENT_BUCKET") == "" && os.Getenv("CARLOS_DEPLOYMENT_DIR") == "" { |
| + return nil |
| + } |
| + return openStore() |
| +} |
| + |
| // openStore mirrors the carlos CLI's ~10-line store-open helper (cli.go); |
| // duplicated here because cmd/console cannot import package main. |
| func openStore() release.Store { |
diff --git a/cmd/console/member.go b/cmd/console/member.go| index 03aed9239..bbf513893 100644 |
| --- a/cmd/console/member.go |
| +++ b/cmd/console/member.go |
| @@ -1,6 +1,7 @@ |
| package main |
| import ( |
| + "context" |
| "flag" |
| "fmt" |
| "log" |
| @@ -77,14 +78,24 @@ func runMember(args []string) { |
| st := openConsoleDB() |
| defer st.Close() |
| + // The bucket is the control plane and the console is not its only minter: |
| + // `carlos accounts create` writes the manifest and the registry row without |
| + // ever opening this database. Catch up BEFORE deciding a sqid does not |
| + // exist, or this verb refuses to administer accounts that demonstrably do. |
| + if _, err := console.MaterializeAccounts(context.Background(), optionalStore(), st); err != nil { |
| + // Not fatal: an unreachable bucket must never be the thing that stops |
| + // an operator removing a membership. |
| + log.Printf("warning: could not read accounts from the deployment bucket: %v", err) |
| + } |
| acct, err := st.AccountBySqid(sqid) |
| if err != nil { |
| log.Fatal(err) |
| } |
| if acct == nil { |
| - log.Fatalf("no account %q: nothing with that sqid in %s — create one with: "+ |
| - "carlos accounts create -name <name>", sqid, envOr("CONSOLE_DB", "/data/console/console.db")) |
| + log.Fatalf("no account %q: nothing with that sqid in %s, and no %s in the deployment bucket — "+ |
| + "create one with: carlos accounts create -name <name>", |
| + sqid, envOr("CONSOLE_DB", "/data/console/console.db"), console.AccountManifestKey(sqid)) |
| } |
| switch { |
diff --git a/cmd/console/member_test.go b/cmd/console/member_test.go| index b5a46d7b9..b786ee3c2 100644 |
| --- a/cmd/console/member_test.go |
| +++ b/cmd/console/member_test.go |
| @@ -7,6 +7,8 @@ import ( |
| "time" |
| "github.com/carlosframework/platform/internal/console" |
| + "github.com/carlosframework/platform/internal/domains" |
| + "github.com/carlosframework/platform/internal/release" |
| ) |
| // `console member` is the box-side membership tool, and it exists because |
| @@ -158,7 +160,7 @@ func TestMemberVerbRejectsBadArguments(t *testing.T) { |
| want []string |
| }{ |
| {"unknown account", []string{"member", "-add", "dev@keymail.dev", "nosuch"}, |
| - []string{"nosuch", "carlos accounts create"}}, |
| + []string{"nosuch", "accounts/nosuch/account.json", "carlos accounts create"}}, |
| {"malformed address", []string{"member", "-add", "dev", "bab"}, |
| []string{"dev", "@"}}, |
| {"unknown role", []string{"member", "-add", "dev@keymail.dev", "bab", "-role", "admin"}, |
| @@ -180,6 +182,45 @@ func TestMemberVerbRejectsBadArguments(t *testing.T) { |
| } |
| } |
| +// `carlos accounts create` mints an account into the BUCKET and the registry |
| +// and never opens the console database. Before materialization, `console |
| +// member -add` refused that sqid — on the same box that had just created it. |
| +func TestMemberVerbAdministersABucketOnlyAccount(t *testing.T) { |
| + bin := buildConsole(t) |
| + tmp := t.TempDir() |
| + dbPath := filepath.Join(tmp, "console.db") |
| + bucket := filepath.Join(tmp, "bucket") |
| + run := runner(t, bin, "CONSOLE_DB="+dbPath, "CARLOS_DEPLOYMENT_DIR="+bucket) |
| + |
| + // Exactly what `carlos accounts create` leaves behind: a manifest, and |
| + // nothing at all in the console's database. |
| + const id = int64(42) |
| + sqid := domains.Sqid(id) |
| + rel := release.NewDirStore(bucket) |
| + if err := console.WriteAccountManifest(t.Context(), rel, id, sqid, "Kass", time.Now()); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + out, err := run("member", "-add", "paul@keymail.dev", sqid, "-role", "owner") |
| + if err != nil { |
| + t.Fatalf("member -add on a bucket-only account: %v\n%s", err, out) |
| + } |
| + if role := roleOf(t, dbPath, id, "paul@keymail.dev"); role != console.RoleOwner { |
| + t.Fatalf("role = %q, want owner", role) |
| + } |
| + // The id came from the manifest, not from a fresh mint: an invented id |
| + // would put the account at a different sqid and re-home everything it owns. |
| + st, err := console.Open(dbPath) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + defer st.Close() |
| + acct, err := st.AccountBySqid(sqid) |
| + if err != nil || acct == nil || acct.ID != id || acct.Name != "Kass" { |
| + t.Fatalf("materialized account = %+v (%v), want id %d named Kass", acct, err, id) |
| + } |
| +} |
| + |
| func roleOf(t *testing.T, dbPath string, accountID int64, address string) string { |
| t.Helper() |
| st, err := console.Open(dbPath) |
diff --git a/internal/console/materialize.go b/internal/console/materialize.go| new file mode 100644 |
| index 000000000..a8ec794a6 |
| --- /dev/null |
| +++ b/internal/console/materialize.go |
| @@ -0,0 +1,104 @@ |
| +package console |
| + |
| +import ( |
| + "context" |
| + "encoding/json" |
| + "log" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/domains" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// MaterializeAccounts teaches the console about accounts it did not mint. |
| +// |
| +// The bucket is the control plane, and the console is NOT the only minter: |
| +// `carlos accounts create` writes accounts/<sqid>/account.json and a registry |
| +// row without ever opening this database — deliberately, because the carlos |
| +// binary has no business inside console state. The consequence, before this, |
| +// was a console that would refuse `member -add … <sqid>` for an account that |
| +// demonstrably exists on the very box that created it. The agent's reconcile |
| +// pass (internal/domains §1) already materializes exactly these manifests into |
| +// the registry; this is the same pass for the console's own tables, reading |
| +// the same objects through the same key parser. |
| +// |
| +// The id in the manifest is TAKEN AS GIVEN. Nothing here ever mints or |
| +// reassigns one: the id is what the sqid encodes, the sqid is in every |
| +// platform domain and every bucket key, and inventing one would re-home an |
| +// account's whole world. A manifest that disagrees with itself — path sqid, |
| +// body sqid and Sqid(id) must all be the same string — is corruption, not a |
| +// new account, and is skipped loudly. |
| +// |
| +// INSERT-only: an account the console already knows keeps its row, name and |
| +// members. Returns how many were newly materialized. |
| +// |
| +// Best-effort by design. A nil store (no CARLOS_DEPLOYMENT_BUCKET/DIR on this |
| +// box) and a failed listing are both "nothing to catch up on", not errors: an |
| +// unreachable bucket must never be what stops an operator removing a |
| +// membership, which is the emergency this whole verb family exists for. |
| +func MaterializeAccounts(ctx context.Context, s release.Store, st *Store) (int, error) { |
| + if s == nil { |
| + return 0, nil |
| + } |
| + keys, err := s.List(ctx, "accounts/") |
| + if err != nil { |
| + return 0, err |
| + } |
| + n := 0 |
| + for _, key := range keys { |
| + sqid, ok := domains.AccountManifestSqid(key) |
| + if !ok { |
| + continue |
| + } |
| + // Cheap short-circuit, and the reason a steady-state pass does no |
| + // writes at all: the console usually already knows every account. |
| + if a, err := st.AccountBySqid(sqid); err != nil { |
| + return n, err |
| + } else if a != nil { |
| + continue |
| + } |
| + body, _, err := s.Get(ctx, key) |
| + if err != nil { |
| + log.Printf("console: skipping %s: read failed: %v", key, err) |
| + continue |
| + } |
| + var m accountManifest |
| + if err := json.Unmarshal(body, &m); err != nil || m.ID <= 0 || m.Sqid == "" { |
| + log.Printf("console: skipping %s: malformed account manifest: %v", key, err) |
| + continue |
| + } |
| + if m.Sqid != sqid || domains.Sqid(m.ID) != sqid { |
| + log.Printf("console: skipping %s: sqid %q/%q is not Sqid(%d)=%q — that is corruption, "+ |
| + "not a new account", key, sqid, m.Sqid, m.ID, domains.Sqid(m.ID)) |
| + continue |
| + } |
| + created := m.CreatedAt |
| + if created.IsZero() { |
| + created = time.Now() |
| + } |
| + added, err := st.EnsureAccount(m.ID, m.Sqid, m.Name, created) |
| + if err != nil { |
| + return n, err |
| + } |
| + if !added { |
| + // The sqid was free but the id was not: some other row already |
| + // holds it. Loud, because two accounts cannot share an id and one |
| + // of these two is wrong. |
| + log.Printf("console: did not materialize %s: account id %d is already taken here", key, m.ID) |
| + continue |
| + } |
| + log.Printf("console: materialized account %s (%s) from the bucket", m.Sqid, m.Name) |
| + n++ |
| + } |
| + return n, nil |
| +} |
| + |
| +// materializeQuietly is the call site every entry point actually wants: catch |
| +// up if we can, log and carry on if we cannot. Separated from |
| +// MaterializeAccounts so the failure policy lives in ONE place rather than |
| +// being re-decided by the server, the member verb and whatever comes next. |
| +func materializeQuietly(ctx context.Context, s release.Store, st *Store) { |
| + if _, err := MaterializeAccounts(ctx, s, st); err != nil { |
| + log.Printf("console: could not materialize accounts from the bucket: %v", err) |
| + } |
| +} |
diff --git a/internal/console/materialize_test.go b/internal/console/materialize_test.go| new file mode 100644 |
| index 000000000..302724968 |
| --- /dev/null |
| +++ b/internal/console/materialize_test.go |
| @@ -0,0 +1,142 @@ |
| +package console |
| + |
| +import ( |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/domains" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// The bucket is the control plane, and the console is not the only minter: |
| +// `carlos accounts create` writes accounts/<sqid>/account.json and a registry |
| +// row without ever opening the console database. Before this, the console |
| +// simply did not know such an account existed — so `console member -add` would |
| +// refuse a sqid that demonstrably exists, on the very box that created it. |
| +func TestMaterializeAccountsFromTheBucket(t *testing.T) { |
| + st := openTestStore(t) |
| + rel := release.NewDirStore(t.TempDir()) |
| + ctx := t.Context() |
| + created := time.Unix(1700000000, 0).UTC() |
| + |
| + const kassID, blogID = int64(793481920157), int64(42) |
| + for _, a := range []struct { |
| + id int64 |
| + name string |
| + }{{kassID, "Kass"}, {blogID, "Blog"}} { |
| + if err := WriteAccountManifest(ctx, rel, a.id, domains.Sqid(a.id), a.name, created); err != nil { |
| + t.Fatal(err) |
| + } |
| + } |
| + |
| + n, err := MaterializeAccounts(ctx, rel, st) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if n != 2 { |
| + t.Errorf("materialized %d accounts, want 2", n) |
| + } |
| + acct, err := st.AccountBySqid(domains.Sqid(kassID)) |
| + if err != nil || acct == nil { |
| + t.Fatalf("AccountBySqid(Kass) = %v %v, want the materialized account", acct, err) |
| + } |
| + // The id in the manifest is taken as GIVEN. Minting a new one here would |
| + // re-home every platform domain the account owns. |
| + if acct.ID != kassID { |
| + t.Errorf("materialized id = %d, want the manifest's %d", acct.ID, kassID) |
| + } |
| + if acct.Name != "Kass" || !acct.CreatedAt.Equal(created) { |
| + t.Errorf("materialized account = %+v, want Kass created %v", acct, created) |
| + } |
| + |
| + // Idempotent: a second pass finds nothing new, which is what makes it safe |
| + // to run on every console start and before every membership operation. |
| + if n, err := MaterializeAccounts(ctx, rel, st); err != nil || n != 0 { |
| + t.Errorf("second pass materialized %d (%v), want 0", n, err) |
| + } |
| +} |
| + |
| +// A manifest whose sqid disagrees with its id is corruption, not a new |
| +// account: materializing it would put an account at a domain that stops |
| +// resolving the moment anything recomputes the sqid. Skipped loudly, never |
| +// guessed at — the same rule internal/domains applies to the same objects. |
| +func TestMaterializeSkipsManifestsThatDisagreeWithThemselves(t *testing.T) { |
| + st := openTestStore(t) |
| + rel := release.NewDirStore(t.TempDir()) |
| + ctx := t.Context() |
| + |
| + // sqid in the path and body agree with each other but not with Sqid(id). |
| + if _, err := rel.Put(ctx, "accounts/wrong/account.json", |
| + []byte(`{"id":42,"sqid":"wrong","name":"Impostor"}`), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + // Body sqid disagrees with the path. |
| + if _, err := rel.Put(ctx, "accounts/"+domains.Sqid(43)+"/account.json", |
| + []byte(`{"id":43,"sqid":"elsewhere","name":"Mislabelled"}`), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := rel.Put(ctx, "accounts/broken/account.json", []byte("{not json"), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := rel.Put(ctx, "accounts/empty/account.json", []byte("{}"), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + // One good manifest alongside: a bad neighbour must not stop the pass. |
| + if err := WriteAccountManifest(ctx, rel, 42, domains.Sqid(42), "Blog", time.Now()); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + n, err := MaterializeAccounts(ctx, rel, st) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if n != 1 { |
| + t.Errorf("materialized %d, want only the well-formed manifest", n) |
| + } |
| + for _, sqid := range []string{"wrong", "elsewhere", "broken", "empty", domains.Sqid(43)} { |
| + if a, _ := st.AccountBySqid(sqid); a != nil { |
| + t.Errorf("materialized %q from a manifest that disagrees with itself: %+v", sqid, a) |
| + } |
| + } |
| + if a, _ := st.AccountBySqid(domains.Sqid(42)); a == nil { |
| + t.Error("the well-formed manifest was not materialized") |
| + } |
| +} |
| + |
| +// Materialization is INSERT-only. An account the console already knows keeps |
| +// its row: the console is where names are edited, the manifest is written |
| +// once, and letting a stale manifest name win would rename accounts behind |
| +// the operator's back. Above all it must never reassign an id. |
| +func TestMaterializeNeverRewritesAKnownAccount(t *testing.T) { |
| + st := openTestStore(t) |
| + rel := release.NewDirStore(t.TempDir()) |
| + ctx := t.Context() |
| + now := time.Unix(1700000000, 0).UTC() |
| + |
| + if err := st.CreateAccount(42, domains.Sqid(42), "Current name", "paul@keymail.dev", now); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := WriteAccountManifest(ctx, rel, 42, domains.Sqid(42), "Stale name", now); err != nil { |
| + t.Fatal(err) |
| + } |
| + if n, err := MaterializeAccounts(ctx, rel, st); err != nil || n != 0 { |
| + t.Fatalf("materialized %d (%v) for an account the console already had", n, err) |
| + } |
| + acct, _ := st.AccountBySqid(domains.Sqid(42)) |
| + if acct == nil || acct.Name != "Current name" || acct.ID != 42 { |
| + t.Errorf("known account was rewritten: %+v", acct) |
| + } |
| + // And its membership survives — materialization touches memberships never. |
| + if role, _ := st.Role(42, "paul@keymail.dev"); role != RoleOwner { |
| + t.Errorf("owner role after materialize = %q, want owner", role) |
| + } |
| +} |
| + |
| +// A deployment with no bucket configured at all still has to work: this is |
| +// best-effort catch-up, not a precondition. |
| +func TestMaterializeWithNoStoreIsANoOp(t *testing.T) { |
| + st := openTestStore(t) |
| + if n, err := MaterializeAccounts(t.Context(), nil, st); err != nil || n != 0 { |
| + t.Errorf("MaterializeAccounts(nil store) = %d %v, want 0 nil", n, err) |
| + } |
| +} |
diff --git a/internal/console/server.go b/internal/console/server.go| index 6d1045044..915d50e1d 100644 |
| --- a/internal/console/server.go |
| +++ b/internal/console/server.go |
| @@ -157,6 +157,14 @@ func NewServer(cfg Config, st *Store) (*Server, error) { |
| s.mux.HandleFunc("GET /healthz", func(w http.ResponseWriter, r *http.Request) { |
| fmt.Fprintln(w, "ok") |
| }) |
| + // Catch up on accounts minted outside this database — `carlos accounts |
| + // create` writes the bucket manifest and the registry row and never opens |
| + // console state. Startup is the natural place: the account switcher, the |
| + // membership gate and every /a/<sqid>/… route resolve through the accounts |
| + // table, so an account the console has not materialized is invisible to |
| + // all of them. Best-effort; a bucket hiccup must not stop the console |
| + // booting. |
| + materializeQuietly(context.Background(), cfg.Releases, st) |
| // One-time backfill: every existing per-app grant becomes a membership of |
| // the ops account, which is where every pre-accounts app already lives. |
| // Idempotent, and cheap enough to run on every start. |
diff --git a/internal/console/store.go b/internal/console/store.go| index 21d2e3cca..ac9869997 100644 |
| --- a/internal/console/store.go |
| +++ b/internal/console/store.go |
| @@ -433,6 +433,30 @@ func (s *Store) CreateAccount(id int64, sqid, name, ownerAddress string, now tim |
| return tx.Commit() |
| } |
| +// EnsureAccount records an account the console did NOT mint — one that already |
| +// exists in the bucket (MaterializeAccounts). added reports whether this call |
| +// created the row. |
| +// |
| +// Unlike CreateAccount it deliberately creates no membership, and that is not |
| +// an oversight: `carlos accounts create` records no owner anywhere, so the |
| +// console has nobody to name. Inventing one — the invoking operator, the first |
| +// user, anyone — would be minting authority out of thin air over an account |
| +// this box knows nothing about. An ownerless row is honest, harmless (no |
| +// membership means no access), and exactly what `console member -add <addr> |
| +// <sqid> -role owner` is for. |
| +// |
| +// INSERT OR IGNORE, never REPLACE: an account the console already knows must |
| +// keep its id, its name and its members. |
| +func (s *Store) EnsureAccount(id int64, sqid, name string, created time.Time) (added bool, err error) { |
| + res, err := s.db.Exec(`INSERT OR IGNORE INTO accounts (id, sqid, name, created_at) VALUES (?, ?, ?, ?)`, |
| + id, sqid, name, created.Unix()) |
| + if err != nil { |
| + return false, err |
| + } |
| + n, err := res.RowsAffected() |
| + return n > 0, err |
| +} |
| + |
| // AccountBySqid returns the account with that public identity, or (nil, nil) |
| // when there is none — "no such account" is an answer, not an error, and the |
| // caller renders it as the same 404 a nonexistent page gets. |
diff --git a/internal/domains/reconcile.go b/internal/domains/reconcile.go| index a38392895..2d0845f46 100644 |
| --- a/internal/domains/reconcile.go |
| +++ b/internal/domains/reconcile.go |
| @@ -84,7 +84,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg |
| // the built-in ops account. |
| known := map[string]bool{OpsSqid: true} // sqid -> materialized |
| for _, key := range acctKeys { |
| - sqid, ok := accountManifestSqid(key) |
| + sqid, ok := AccountManifestSqid(key) |
| if !ok { |
| continue |
| } |
| @@ -319,10 +319,16 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg |
| // deployment has it, console or not. |
| var OpsSqid = Sqid(1) |
| -// accountManifestSqid reports whether key is an account manifest |
| +// AccountManifestSqid reports whether key is an account manifest |
| // (accounts/<sqid>/account.json, exactly one path level deep) and which sqid |
| // its PATH claims. |
| -func accountManifestSqid(key string) (string, bool) { |
| +// |
| +// Exported because the console materializes accounts from the SAME objects |
| +// this reconcile pass does. One parser, not two: if the two ever disagreed |
| +// about what counts as a manifest, the console and the registry would hold |
| +// different account sets on the same box — and the account set is what tells a |
| +// qualified bucket key from a legacy one. |
| +func AccountManifestSqid(key string) (string, bool) { |
| rest, ok := strings.CutPrefix(key, "accounts/") |
| if !ok { |
| return "", false |