amadan

carlos / platform Public

domains: an account stays known when only its local row fails to write

known is what makes an apps/<sqid>/<app>/... key path unambiguous. It
drives the trash-marker scan, the app->account linkage and the
canary-pointer scan — every part of the pass that reads a bucket key.

Setting it only after a successful PutAccount meant a transient SQLite
error un-knew the account for the rest of the pass: this box would then
read every one of that tenant's keys as legacy and silently skip their
trash markers, routes and canary pointers until some later pass
happened to get the row written. The manifest is the authority on which
accounts exist; the registry row is only this box's copy of that fact.

Staying known costs a stale `carlos accounts list` row on one box,
which the next pass fixes.

The loop moves into materializeAccounts so the invariant is reachable
by a test — one vantage, reg never nil, no new exported API.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Paul Campbell pushed by paul@keymail.dev ad78721fc0e8d145e1703b205292d5f643910755 parent cef3dd6
2 files changed, +134 −53
  • internal/domains/reconcile.go +77 −49
  • internal/domains/reconcile_internal_test.go +57 −4
diff --git a/internal/domains/reconcile.go b/internal/domains/reconcile.go
index c7df26c2b..a812f016b 100644
--- a/internal/domains/reconcile.go
+++ b/internal/domains/reconcile.go
@@ -92,55 +92,7 @@ func Reconcile(ctx context.Context, s release.Store, reg *registry.Registry, cfg
}
// --- 1. account materialization ------------------------------------------
- // The console originates account truth as accounts/<sqid>/account.json in
- // the bucket; this is the only place it becomes registry state, and the id
- // in the manifest is TAKEN AS GIVEN — never assigned here. That is the
- // whole durability property: a rebuilt registry, or a second box, ends up
- // with byte-identical account identity, so platform domains never re-home.
- // A deployment with no console has no manifests: everything stays under
- // the built-in ops account.
- known := map[string]bool{OpsSqid: true} // sqid -> materialized
- for _, key := range acctKeys {
- sqid, ok := AccountManifestSqid(key)
- if !ok {
- continue
- }
- body, _, err := s.Get(ctx, key)
- if err != nil {
- log.Printf("domains: skipping %s: read failed: %v", key, err)
- continue
- }
- var m AccountManifest
- if err := json.Unmarshal(body, &m); err != nil || m.ID <= 0 || m.Sqid == "" {
- log.Printf("domains: skipping %s: malformed account manifest: %v", key, err)
- continue
- }
- // Three things must agree: the key path, the body's sqid, and the
- // frozen encoding of the body's id. A manifest that disagrees with
- // itself is a lie about identity — materializing it would put an
- // account at a domain that stops resolving the moment anything
- // recomputes the sqid — so it is skipped, loudly, never guessed at.
- if m.Sqid != sqid || Sqid(m.ID) != sqid {
- log.Printf("domains: skipping %s: sqid %q/%q is not Sqid(%d)=%q", key, sqid, m.Sqid, m.ID, Sqid(m.ID))
- continue
- }
- // A tombstoned manifest is materialized nowhere and, if this registry
- // still carries the account from before it was retired, un-materialized
- // here. Leaving the row would keep the account listed by `carlos
- // accounts list` on every box forever — a deletion that only ever took
- // effect in the console that performed it.
- if m.Deleted() {
- if err := reg.DeleteAccount(m.Sqid); err != nil {
- log.Printf("domains: retire account %s: %v", sqid, err)
- }
- continue
- }
- if err := reg.PutAccount(m.ID, m.Sqid, m.Name); err != nil {
- log.Printf("domains: materialize account %s: %v", sqid, err)
- continue
- }
- known[sqid] = true
- }
+ known := materializeAccounts(ctx, s, acctKeys, reg)
// --- 1c. trashed apps ----------------------------------------------------
// A valid trash marker (apps/<sqid>/<app>/deleted.json) is the owner's
@@ -572,6 +524,82 @@ func AccountManifestSqid(key string) (string, bool) {
return sqid, true
}
+// materializeAccounts turns the deployment's account manifests into this
+// box's registry account rows, and returns the set of sqids that exist.
+//
+// The return value is the point: known is what makes an apps/<sqid>/<app>/…
+// key path unambiguous (appKeyOwner), so it drives the trash-marker scan,
+// the app->account linkage and the canary-pointer scan — every part of the
+// pass that reads a bucket key. It is a function rather than an inline loop
+// so that invariant is reachable by a test; there is one vantage, and reg is
+// never nil.
+func materializeAccounts(ctx context.Context, s release.Store, acctKeys []string, reg *registry.Registry) map[string]bool {
+ // The console originates account truth as accounts/<sqid>/account.json in
+ // the bucket; this is the only place it becomes registry state, and the id
+ // in the manifest is TAKEN AS GIVEN — never assigned here. That is the
+ // whole durability property: a rebuilt registry, or a second box, ends up
+ // with byte-identical account identity, so platform domains never re-home.
+ // A deployment with no console has no manifests: everything stays under
+ // the built-in ops account.
+ known := map[string]bool{OpsSqid: true} // sqid -> materialized
+ for _, key := range acctKeys {
+ sqid, ok := AccountManifestSqid(key)
+ if !ok {
+ continue
+ }
+ body, _, err := s.Get(ctx, key)
+ if err != nil {
+ log.Printf("domains: skipping %s: read failed: %v", key, err)
+ continue
+ }
+ var m AccountManifest
+ if err := json.Unmarshal(body, &m); err != nil || m.ID <= 0 || m.Sqid == "" {
+ log.Printf("domains: skipping %s: malformed account manifest: %v", key, err)
+ continue
+ }
+ // Three things must agree: the key path, the body's sqid, and the
+ // frozen encoding of the body's id. A manifest that disagrees with
+ // itself is a lie about identity — materializing it would put an
+ // account at a domain that stops resolving the moment anything
+ // recomputes the sqid — so it is skipped, loudly, never guessed at.
+ if m.Sqid != sqid || Sqid(m.ID) != sqid {
+ log.Printf("domains: skipping %s: sqid %q/%q is not Sqid(%d)=%q", key, sqid, m.Sqid, m.ID, Sqid(m.ID))
+ continue
+ }
+ // A tombstoned manifest is materialized nowhere and, if this registry
+ // still carries the account from before it was retired, un-materialized
+ // here. Leaving the row would keep the account listed by `carlos
+ // accounts list` on every box forever — a deletion that only ever took
+ // effect in the console that performed it.
+ if m.Deleted() {
+ if err := reg.DeleteAccount(m.Sqid); err != nil {
+ log.Printf("domains: retire account %s: %v", sqid, err)
+ }
+ continue
+ }
+ // KNOWN FIRST, then the write. The manifest is the authority on
+ // which accounts EXIST; the registry row is only this box's local
+ // copy of that fact, and the two answer different questions. known
+ // answers the first — it is what tells a qualified bucket key
+ // (apps/<sqid>/<app>/…) from a legacy one, and nothing else.
+ //
+ // So a failed row-write must not un-know the account. If it did,
+ // every one of that account's keys would read as legacy for the rest
+ // of the pass (appKeyOwner returns !ok), and this box would silently
+ // skip the account's trash markers, route linkage and canary
+ // pointers — a transient SQLite error turning into "this tenant's
+ // apps stop reconciling" until some later pass happens to write the
+ // row. Staying known costs only a stale `carlos accounts list` row on
+ // this one box, which the next pass fixes.
+ known[sqid] = true
+ if err := reg.PutAccount(m.ID, m.Sqid, m.Name); err != nil {
+ log.Printf("domains: materialize account %s: %v", sqid, err)
+ continue
+ }
+ }
+ return known
+}
+
// appKeyOwner reads the app→account linkage straight out of a bucket key:
// apps/<sqid>/<app>/… belongs to that account. known is the set of
// materialized account sqids, which is what makes a key path unambiguous —
diff --git a/internal/domains/reconcile_internal_test.go b/internal/domains/reconcile_internal_test.go
index 4075fd2a9..b44c18ea6 100644
--- a/internal/domains/reconcile_internal_test.go
+++ b/internal/domains/reconcile_internal_test.go
@@ -1,18 +1,22 @@
package domains
-// Internal-package tests for hostCertExpiry: reconcile_test.go
-// is package domains_test and cannot reach these unexported helpers, so this
-// file follows slugname_test.go's existing convention of a same-package test
-// file living alongside it.
+// Internal-package tests for reconcile.go's unexported helpers
+// (hostCertExpiry, mintedRoute, materializeAccounts): reconcile_test.go
+// is package domains_test and cannot reach these, so this file follows
+// slugname_test.go's existing convention of a same-package test file living
+// alongside it.
import (
"bytes"
+ "context"
"crypto/ecdsa"
"crypto/elliptic"
"crypto/rand"
"crypto/x509"
"crypto/x509/pkix"
"encoding/pem"
+ "io"
+ "log"
"math/big"
"os"
"path/filepath"
@@ -20,6 +24,7 @@ import (
"time"
"github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
)
// writeWildcardPEM writes a self-signed cert for name valid until notAfter,
@@ -152,3 +157,51 @@ func TestMintedRouteCarriesRegionAndDropsSocketWhenFleetPlaced(t *testing.T) {
t.Fatalf("mintedRoute Socket = %q, want %q — a non-fleet source's socket is still copied", rt2.Socket, src2.Socket)
}
}
+
+// A failed LOCAL row write does not unmake an account. The manifest is the
+// authority on which accounts exist; the registry row is only this box's copy
+// of that fact. known is what makes an apps/<sqid>/<app>/… key path
+// unambiguous, so dropping the sqid on a write failure would make this box
+// read every one of that account's keys as legacy for the rest of the pass —
+// silently skipping its trash markers, route linkage and canary pointers
+// until some later pass happened to get the row written.
+func TestMaterializeAccountsKeepsAnAccountKnownWhenTheRowWriteFails(t *testing.T) {
+ log.SetOutput(io.Discard)
+ t.Cleanup(func() { log.SetOutput(os.Stderr) })
+
+ ctx := context.Background()
+ s := release.NewDirStore(t.TempDir())
+ const id = 793481920157
+ sqid := Sqid(id)
+ body := []byte(`{"id":793481920157,"sqid":"` + sqid + `","name":"Kass","created_at":"2026-08-02T09:00:00Z"}`)
+ if _, err := s.Put(ctx, "accounts/"+sqid+"/account.json", body, ""); err != nil {
+ t.Fatal(err)
+ }
+ acctKeys, err := s.List(ctx, "accounts/")
+ if err != nil {
+ t.Fatal(err)
+ }
+
+ reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := reg.Close(); err != nil { // every write from here on fails
+ t.Fatal(err)
+ }
+ if err := reg.PutAccount(id, sqid, "Kass"); err == nil {
+ t.Fatal("a closed registry accepted PutAccount — this test injects nothing")
+ }
+
+ known := materializeAccounts(ctx, s, acctKeys, reg)
+
+ if !known[sqid] {
+ t.Fatalf("account %s missing from the set: %v — a failed local row write does not unmake an account whose identity already checked out", sqid, known)
+ }
+ // And the consequence that actually matters: the account's app keys still
+ // read as qualified, so the rest of the pass still sees its objects.
+ acct, app, _, ok := appKeyOwner("apps/"+sqid+"/kass/deleted.json", known)
+ if !ok || acct != sqid || app != "kass" {
+ t.Fatalf("appKeyOwner = (%q, %q, %v); want the key read as qualified under %s", acct, app, ok, sqid)
+ }
+}