domains: fold the claim set from the listing's stamps, not a GET per claim (#173)
ListClaims lists domains/ and then reads every object under it. Three
things do that repeatedly: every app Settings render, every
`carlos domains list`, and every box's ReconcileAliases — the last one on
the FAST-CONVERGE leg, which runs on every serial bump, a seconds cadence,
ahead of the adopt leg inside a two-second convergence promise.
At tens of claims that is invisible. At the hundreds titogo's custom
domains are heading for it is a slow page and a steady drip of bucket
reads on every box in the fleet, forever. It is the shape PR #134 already
paid for once on releases.
The fix needs no new object and no new writer, because the listing already
carries the answer. S3 returns LastModified with every ListObjectsV2
entry, DirStore has the file mtime, and release.ModLister already
surfaces both — List simply throws it away. A claim whose stamp has not
moved has not changed, so its body does not need re-reading. Steady state:
one ListMod and zero GETs. The pass after a claim changes reads exactly
the claims that changed.
No TTL and no invalidation call, deliberately. Two exact rules do the
work: an entry is reused only when its stamp is unchanged, and the entry
map is replaced whole each fold — so a detached claim leaves the cache on
the same pass it leaves the listing, with nothing to remember to call.
Three bounds, each of which is a bug without it:
- modSettle. S3 stamps have one-second granularity, so a claim read and
then rewritten inside the same second carries an unchanged-looking
stamp. Caching that read would serve the old body until something
else moved the stamp — and nothing edits a settled claim, so for that
hostname "until" means never.
- A READ failure is never remembered, only a DECODE refusal. The same
bytes refuse to parse the same way every pass, so remembering that
saves a pointless GET forever; a network blip remembered would hide a
perfectly good claim on exactly the same "forever" timescale.
- A store with no ListMod — the fleet store door's client, every test
fake — degrades to the plain fold, not to a weaker time-based cache.
A customer fleet box is the hardest place to notice a stale claim set.
ReconcileAliases takes the cache explicitly; nil means today's behaviour.
internal/host holds one process-lifetime cache shared by both alias legs,
following withheldAliasSaid's precedent in the same file — one agent per
process, both legs asking about the same deployment's claims, so two
caches would double the cold reads for nothing.
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
10 files changed,
+599
−43
internal/console/apidomains.go+1 −1internal/console/appsettings.go+1 −2internal/console/server.go+7 −2internal/domains/aliases.go+7 −2internal/domains/aliases_test.go+20 −20internal/domains/claimcache.go+136 −0internal/domains/claimcache_test.go+374 −0internal/domains/claims.go+35 −14internal/host/agent.go+2 −2internal/host/edgesfile.go+16 −0
CI passed — run details
diff --git a/internal/console/apidomains.go b/internal/console/apidomains.go| index 5d78f5b13..6991285a0 100644 |
| --- a/internal/console/apidomains.go |
| +++ b/internal/console/apidomains.go |
| @@ -677,7 +677,7 @@ func (s *Server) handleAPIDomainList(w http.ResponseWriter, r *http.Request) { |
| // cross-account uniqueness gate), so the tenant boundary is this filter |
| // and nothing else: every row that reaches the answer matched BOTH this |
| // account's sqid and this app. |
| - claims, err := domains.ListClaims(ctx, s.cfg.Releases) |
| + claims, err := s.claims.List(ctx, s.cfg.Releases) |
| if err != nil { |
| log.Printf("console: api domains list %s/%s: %v", acct.Sqid, app, err) |
| jsonError(w, http.StatusBadGateway, "could not list the hostnames") |
diff --git a/internal/console/appsettings.go b/internal/console/appsettings.go| index 1c1431dec..88a16d976 100644 |
| --- a/internal/console/appsettings.go |
| +++ b/internal/console/appsettings.go |
| @@ -38,7 +38,6 @@ import ( |
| "strings" |
| "time" |
| - "github.com/carlosframework/platform/internal/domains" |
| "github.com/carlosframework/platform/internal/mail" |
| "github.com/carlosframework/platform/internal/objstore" |
| "github.com/carlosframework/platform/internal/release" |
| @@ -432,7 +431,7 @@ func (s *Server) settingsDomains(ctx context.Context, acct *Account, app string, |
| for i, d := range out { |
| have[strings.ToLower(d.Domain)] = i |
| } |
| - claims, err := domains.ListClaims(ctx, s.cfg.Releases) |
| + claims, err := s.claims.List(ctx, s.cfg.Releases) |
| if err != nil { |
| log.Printf("console: settings claims %s/%s: %v", acct.Sqid, app, err) |
| return out |
diff --git a/internal/console/server.go b/internal/console/server.go| index 5328ba156..15b42874e 100644 |
| --- a/internal/console/server.go |
| +++ b/internal/console/server.go |
| @@ -580,8 +580,13 @@ type Config struct { |
| // Server is the console's http.Handler. |
| type Server struct { |
| - cfg Config |
| - store *Store |
| + cfg Config |
| + store *Store |
| + // claims folds the deployment's domain claims for the read paths — the |
| + // Settings render and the domains list door — reading only the claims |
| + // whose bucket stamp moved since the last fold (#173). Process-lifetime |
| + // and safe for concurrent handlers; see domains.ClaimCache. |
| + claims domains.ClaimCache |
| km *Keymail |
| mux *http.ServeMux |
| instances *instanceFetcher |
diff --git a/internal/domains/aliases.go b/internal/domains/aliases.go| index 77b23ce7b..21530eb4c 100644 |
| --- a/internal/domains/aliases.go |
| +++ b/internal/domains/aliases.go |
| @@ -40,8 +40,13 @@ const maxWildcardDomains = 32 |
| // no instance record to read. nil means "nothing known" and is PERMISSIVE by |
| // contract — a bucket the fold could not read must never break ordinary |
| // attach. |
| -func ReconcileAliases(ctx context.Context, s release.Store, reg *registry.Registry, now func() time.Time, deploymentRoutes map[string]bool) error { |
| - claims, err := ListClaims(ctx, s) |
| +// cache, when non-nil, folds the claims through a ClaimCache instead of |
| +// re-reading every claim body. This pass runs on the fast-converge leg — a |
| +// seconds cadence, on every serial bump — so the difference is one bucket |
| +// call against one plus a GET per claim in the deployment, every few |
| +// seconds, on every box. nil is legal and means today's uncached fold. |
| +func ReconcileAliases(ctx context.Context, s release.Store, reg *registry.Registry, now func() time.Time, deploymentRoutes map[string]bool, cache *ClaimCache) error { |
| + claims, err := cache.List(ctx, s) |
| if err != nil { |
| return err |
| } |
diff --git a/internal/domains/aliases_test.go b/internal/domains/aliases_test.go| index 2507395f8..d51f64c4c 100644 |
| --- a/internal/domains/aliases_test.go |
| +++ b/internal/domains/aliases_test.go |
| @@ -77,7 +77,7 @@ func TestReconcileAliasesMaterializes(t *testing.T) { |
| putClaim(t, s, Claim{ |
| Host: "tickets.example.com", Account: "bab", App: "kass", Route: "canary/live", |
| }) |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -103,7 +103,7 @@ func TestReconcileAliasesOwnershipGate(t *testing.T) { |
| putClaim(t, s, Claim{ |
| Host: "tickets.example.com", Account: "bab", App: "somethingelse", Route: "canary/live", |
| }) |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -129,7 +129,7 @@ func TestReconcileAliasesOwnershipGateAccountMismatch(t *testing.T) { |
| putClaim(t, s, Claim{ |
| Host: "tickets.example.com", Account: "bab", App: "kass", Route: "canary/live", |
| }) |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -150,7 +150,7 @@ func TestReconcileAliasesRoutesWin(t *testing.T) { |
| putClaim(t, s, Claim{ |
| Host: "tickets.example.com", Account: "bab", App: "kass", Route: "canary/live", |
| }) |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -168,7 +168,7 @@ func TestReconcileAliasesMissingRouteSkips(t *testing.T) { |
| putClaim(t, s, Claim{ |
| Host: "tickets.example.com", Account: "bab", App: "kass", Route: "canary/live", |
| }) |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -192,7 +192,7 @@ func TestReconcileAliasesRemovesOrphans(t *testing.T) { |
| t.Fatal(err) |
| } |
| // No claim backs orphan.example.com. |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -213,7 +213,7 @@ func TestReconcileAliasesRouteless(t *testing.T) { |
| putClaim(t, s, Claim{ |
| Host: "tickets.example.com", Account: "bab", App: "kass", |
| }) |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -233,7 +233,7 @@ func TestReconcileAliasesIdempotent(t *testing.T) { |
| putClaim(t, s, Claim{ |
| Host: "tickets.example.com", Account: "bab", App: "kass", Route: "canary/live", |
| }) |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| first, err := reg.ListAliases() |
| @@ -247,7 +247,7 @@ func TestReconcileAliasesIdempotent(t *testing.T) { |
| // Second pass, later clock: CreatedAt must be preserved, not bumped. |
| later := func() time.Time { return time.Unix(1755000000+3600, 0) } |
| - if err := ReconcileAliases(context.Background(), s, reg, later, nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, later, nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| second, err := reg.ListAliases() |
| @@ -284,7 +284,7 @@ func TestReconcileWildcardCreatedAtStable(t *testing.T) { |
| Route: "woodstar-relay.bab.oncarlos.com", CreatedAt: "2026-08-13T00:00:00Z", |
| Wildcard: true, Catchall: true}) |
| - if err := ReconcileAliases(context.Background(), s, reg, func() time.Time { return time.Unix(1000, 0) }, nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, func() time.Time { return time.Unix(1000, 0) }, nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| first, err := reg.ListWildcardDomains() |
| @@ -300,7 +300,7 @@ func TestReconcileWildcardCreatedAtStable(t *testing.T) { |
| // not bumped — a bump here would move Fingerprint's wildcard_domains |
| // section every poll and force a reload→refresh loop with nothing |
| // actually changed. |
| - if err := ReconcileAliases(context.Background(), s, reg, func() time.Time { return time.Unix(1000+3600, 0) }, nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, func() time.Time { return time.Unix(1000+3600, 0) }, nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| second, err := reg.ListWildcardDomains() |
| @@ -335,7 +335,7 @@ func TestReconcileDerivesWildcardState(t *testing.T) { |
| putClaim(t, s, Claim{Host: "bad host.example", Account: "bab", App: "woodstar-relay", |
| Route: "woodstar-relay.bab.oncarlos.com", CreatedAt: "2026-08-13T00:00:00Z", Wildcard: true, Catchall: true}) |
| - if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(1000, 0) }, nil); err != nil { |
| + if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(1000, 0) }, nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| @@ -359,7 +359,7 @@ func TestReconcileDerivesWildcardState(t *testing.T) { |
| // Claim removed => derived state removed (pass-owned). |
| deleteClaim(t, s, "woodstar.app") |
| - if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(2000, 0) }, nil); err != nil { |
| + if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(2000, 0) }, nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| wds, _ = reg.ListWildcardDomains() |
| @@ -392,7 +392,7 @@ func TestReconcileWildcardHostCollisionMismatchedTenant(t *testing.T) { |
| Route: "primary.bab.oncarlos.com", CreatedAt: "2026-08-13T00:00:00Z", |
| Wildcard: true, Catchall: true}) |
| - if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(1000, 0) }, nil); err != nil { |
| + if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(1000, 0) }, nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| wds, err := reg.ListWildcardDomains() |
| @@ -427,7 +427,7 @@ func TestReconcileWildcardCap(t *testing.T) { |
| Route: "primary.bab.oncarlos.com", CreatedAt: "2026-08-13T00:00:00Z", Wildcard: true}) |
| } |
| - if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(1000, 0) }, nil); err != nil { |
| + if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(1000, 0) }, nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| wds, err := reg.ListWildcardDomains() |
| @@ -454,7 +454,7 @@ func TestReconcileWildcardEmptyRouteClaim(t *testing.T) { |
| putClaim(t, s, Claim{Host: "story.example", Account: "bab", App: "story", |
| CreatedAt: "2026-08-13T00:00:00Z", Wildcard: true, Catchall: true}) |
| - if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(1000, 0) }, nil); err != nil { |
| + if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(1000, 0) }, nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| wds, _ := reg.ListWildcardDomains() |
| @@ -491,7 +491,7 @@ func TestReconcileAliasesRefusesAHostAnotherBoxServesAsARoute(t *testing.T) { |
| // go.tito.io is a real, non-alias route on some OTHER box in the fleet. |
| deploymentRoutes := map[string]bool{"go.tito.io": true} |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), deploymentRoutes); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), deploymentRoutes, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -524,7 +524,7 @@ func TestReconcileAliasesRemovesAnAliasThatNowCollidesWithADeploymentRoute(t *te |
| deploymentRoutes := map[string]bool{"go.tito.io": true} |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), deploymentRoutes); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), deploymentRoutes, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -553,7 +553,7 @@ func TestReconcileAliasesStillMaterializesWhenTheRouteSetIsUnknown(t *testing.T) |
| Host: "shop.customer.com", Account: "bab", App: "kass", Route: "canary/live", |
| }) |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -593,7 +593,7 @@ func TestReconcileAliasesStillDerivesTheCatchAllForAFirstRouteClaim(t *testing.T |
| // The claimant's own box publishes woodstar.app as a REAL route. |
| deploymentRoutes := map[string]bool{"woodstar.app": true} |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), deploymentRoutes); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), deploymentRoutes, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
diff --git a/internal/domains/claimcache.go b/internal/domains/claimcache.go| new file mode 100644 |
| index 000000000..b64c5f882 |
| --- /dev/null |
| +++ b/internal/domains/claimcache.go |
| @@ -0,0 +1,136 @@ |
| +package domains |
| + |
| +import ( |
| + "context" |
| + "sync" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// ClaimCache makes the claim fold cost one bucket call in the steady state |
| +// instead of one plus a GET per claim (#173, #165). |
| +// |
| +// The problem it solves. ListClaims lists domains/ and then reads every |
| +// object under it, and three things do that repeatedly: every app Settings |
| +// render, every `carlos domains list`, and every box's ReconcileAliases pass |
| +// — the last one on a five-minute floor, ahead of the adopt leg inside a |
| +// two-second convergence promise. At tens of claims that is invisible. At |
| +// the hundreds titogo's custom domains are heading for it is a slow page and |
| +// a steady drip of bucket reads on every box in the fleet, forever. |
| +// |
| +// The fix needs no new object and no new writer, because the listing already |
| +// carries the answer: S3 returns LastModified with every ListObjectsV2 entry |
| +// and DirStore has the file mtime, both of which release.ModLister already |
| +// surfaces and List throws away. A claim whose last-modified time has not |
| +// moved has not changed, so its body does not need re-reading. The steady |
| +// state is therefore ONE ListMod and zero GETs, and the first pass after a |
| +// claim changes reads exactly the claims that changed. |
| +// |
| +// This is a cache with no TTL and no invalidation call, which is the point: |
| +// it is not guessing about staleness, it is being told. Two exact rules do |
| +// all the work — an entry is reused only when its stamp is unchanged, and an |
| +// entry whose key has left the listing is dropped, which is the detach path. |
| +// |
| +// A store that cannot answer ListMod (the fleet store door's client, every |
| +// test fake) degrades to a plain ListClaims with no caching at all. That is |
| +// deliberate: a fallback that cached on time instead would be a different, |
| +// weaker contract wearing the same name, and the one place it would be used |
| +// is the one place — a customer fleet box — where a stale claim set is |
| +// hardest to notice. |
| +// |
| +// Safe for concurrent use. The zero value is ready, and a nil *ClaimCache |
| +// is legal and means "no caching" — so a caller that has no cache to offer |
| +// passes nil rather than every call site growing a branch. |
| +type ClaimCache struct { |
| + mu sync.Mutex |
| + entries map[string]cachedClaim |
| +} |
| + |
| +type cachedClaim struct { |
| + mod time.Time |
| + claim Claim |
| + // skipped records that this key was listed but produced no claim for a |
| + // DETERMINISTIC reason — unparseable, or a schema newer than this |
| + // binary. Cached like a hit so a permanently undecodable object does not |
| + // earn a GET on every pass forever. A key whose READ failed is never |
| + // cached at all: see readClaim's firm return. |
| + skipped bool |
| +} |
| + |
| +// modSettle is how recently an object may have been written before its |
| +// stamp stops being trustworthy as an unchanged-marker. |
| +// |
| +// S3 last-modified has one-second granularity. Without this bound there is a |
| +// real lost update: read a claim at T and cache stamp T, have it rewritten |
| +// later in the same second T, and every later listing shows stamp T — which |
| +// this cache would read as "unchanged" forever. Refusing to cache anything |
| +// stamped within the last two seconds closes it, at the cost of re-reading a |
| +// just-changed claim once or twice more. Claims change on human timescales, |
| +// so that cost is approximately never paid. |
| +const modSettle = 2 * time.Second |
| + |
| +// List folds the deployment's claims, reading only the ones that changed. |
| +// |
| +// The returned slice is fresh on every call and safe for the caller to keep; |
| +// Claim is a value type with no reference fields, so cached entries cannot |
| +// be mutated through it. |
| +func (c *ClaimCache) List(ctx context.Context, s release.Store) ([]Claim, error) { |
| + lister, ok := s.(release.ModLister) |
| + if c == nil || !ok { |
| + return ListClaims(ctx, s) |
| + } |
| + listed, err := lister.ListMod(ctx, ClaimsPrefix) |
| + if err != nil { |
| + return nil, err |
| + } |
| + now := time.Now() |
| + c.mu.Lock() |
| + defer c.mu.Unlock() |
| + next := make(map[string]cachedClaim, len(listed)) |
| + claims := make([]Claim, 0, len(listed)) |
| + for _, km := range listed { |
| + if hit, ok := c.entries[km.Key]; ok && hit.mod.Equal(km.Mod) { |
| + // Carried forward WITHOUT re-checking modSettle: the bound |
| + // applies when an entry is admitted, so anything already in the |
| + // map was settled when it went in. |
| + next[km.Key] = hit |
| + if !hit.skipped { |
| + claims = append(claims, hit.claim) |
| + } |
| + continue |
| + } |
| + claim, ok, firm := readClaim(ctx, s, km.Key) |
| + if ok { |
| + claims = append(claims, claim) |
| + } |
| + if !firm { |
| + // The read itself failed. Remembering that would hide a good |
| + // claim until something moved its stamp — and nothing edits a |
| + // settled claim, so that is indistinguishable from forever. |
| + continue |
| + } |
| + if now.Sub(km.Mod) < modSettle { |
| + // Too fresh to trust as an unchanged-marker. Used, not kept. |
| + continue |
| + } |
| + next[km.Key] = cachedClaim{mod: km.Mod, claim: claim, skipped: !ok} |
| + } |
| + // Whatever is not in next is not in the bucket: replacing the map whole |
| + // IS the eviction, so a detached claim leaves the cache on the same pass |
| + // it leaves the listing, with no separate invalidation to forget. |
| + c.entries = next |
| + return claims, nil |
| +} |
| + |
| +// Reset drops everything. Nothing in this package needs it; it exists so a |
| +// caller whose store changed underneath it (a test, a reconfiguration) can |
| +// say so explicitly rather than construct a new cache and lose the field. |
| +func (c *ClaimCache) Reset() { |
| + if c == nil { |
| + return |
| + } |
| + c.mu.Lock() |
| + defer c.mu.Unlock() |
| + c.entries = nil |
| +} |
diff --git a/internal/domains/claimcache_test.go b/internal/domains/claimcache_test.go| new file mode 100644 |
| index 000000000..985dffad1 |
| --- /dev/null |
| +++ b/internal/domains/claimcache_test.go |
| @@ -0,0 +1,374 @@ |
| +package domains |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "os" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// countingStore counts the GETs the fold spends. That count IS the property |
| +// under test — the whole point of the cache is that a claim whose bucket |
| +// stamp has not moved costs nothing to fold again. |
| +type countingStore struct { |
| + release.Store |
| + gets int |
| +} |
| + |
| +func (c *countingStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + c.gets++ |
| + return c.Store.Get(ctx, key) |
| +} |
| + |
| +func (c *countingStore) ListMod(ctx context.Context, prefix string) ([]release.KeyMod, error) { |
| + return c.Store.(release.ModLister).ListMod(ctx, prefix) |
| +} |
| + |
| +func seedClaim(t *testing.T, s release.Store, host, app string) { |
| + t.Helper() |
| + if _, _, err := AttachClaim(context.Background(), s, Claim{Host: host, Account: "acct", App: app}); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +// backdate moves a claim's bucket stamp out of the modSettle window, which |
| +// is what a claim written more than a couple of seconds ago looks like. Real |
| +// claims change on human timescales; a test cannot afford to sleep for them. |
| +func backdate(t *testing.T, dir, host string) { |
| + t.Helper() |
| + old := time.Now().Add(-time.Hour) |
| + if err := os.Chtimes(dir+"/"+ClaimKey(host), old, old); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +func TestClaimCacheReReadsNothingWhenNothingChanged(t *testing.T) { |
| + dir := t.TempDir() |
| + s := &countingStore{Store: release.NewDirStore(dir)} |
| + ctx := context.Background() |
| + for _, h := range []string{"a.customer.test", "b.customer.test", "c.customer.test"} { |
| + seedClaim(t, s, h, "kass") |
| + backdate(t, dir, h) |
| + } |
| + s.gets = 0 |
| + |
| + var c ClaimCache |
| + first, err := c.List(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(first) != 3 { |
| + t.Fatalf("first fold = %+v, want 3 claims", first) |
| + } |
| + cold := s.gets |
| + if cold != 3 { |
| + t.Fatalf("cold fold spent %d gets, want one per claim", cold) |
| + } |
| + |
| + s.gets = 0 |
| + second, err := c.List(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(second) != 3 { |
| + t.Fatalf("warm fold = %+v, want the same 3 claims", second) |
| + } |
| + if s.gets != 0 { |
| + t.Errorf("warm fold spent %d gets, want 0 — an unmoved stamp must cost nothing", s.gets) |
| + } |
| +} |
| + |
| +// The cache must be invisible: what it returns has to be what the uncached |
| +// fold returns, claim for claim, or a hostname appears on one read path and |
| +// vanishes on another. |
| +func TestClaimCacheAgreesWithTheUncachedFold(t *testing.T) { |
| + dir := t.TempDir() |
| + s := release.NewDirStore(dir) |
| + ctx := context.Background() |
| + for _, h := range []string{"a.customer.test", "b.customer.test"} { |
| + seedClaim(t, s, h, "kass") |
| + backdate(t, dir, h) |
| + } |
| + var c ClaimCache |
| + if _, err := c.List(ctx, s); err != nil { // prime |
| + t.Fatal(err) |
| + } |
| + cached, err := c.List(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + plain, err := ListClaims(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(cached) != len(plain) { |
| + t.Fatalf("cached fold has %d claims, uncached has %d", len(cached), len(plain)) |
| + } |
| + byHost := map[string]Claim{} |
| + for _, p := range plain { |
| + byHost[p.Host] = p |
| + } |
| + for _, got := range cached { |
| + want, ok := byHost[got.Host] |
| + if !ok { |
| + t.Errorf("cached fold invented %s", got.Host) |
| + continue |
| + } |
| + if got != want { |
| + t.Errorf("%s: cached %+v, uncached %+v", got.Host, got, want) |
| + } |
| + } |
| +} |
| + |
| +// A detached claim leaves the listing, and leaving the listing IS the |
| +// eviction — there is no separate invalidation call to forget to make. |
| +func TestClaimCacheDropsADetachedClaim(t *testing.T) { |
| + dir := t.TempDir() |
| + s := release.NewDirStore(dir) |
| + ctx := context.Background() |
| + seedClaim(t, s, "a.customer.test", "kass") |
| + seedClaim(t, s, "b.customer.test", "kass") |
| + backdate(t, dir, "a.customer.test") |
| + backdate(t, dir, "b.customer.test") |
| + |
| + var c ClaimCache |
| + if got, err := c.List(ctx, s); err != nil || len(got) != 2 { |
| + t.Fatalf("first fold = %+v, %v", got, err) |
| + } |
| + if _, err := DetachClaim(ctx, s, "b.customer.test", "acct", "kass"); err != nil { |
| + t.Fatal(err) |
| + } |
| + got, err := c.List(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(got) != 1 || got[0].Host != "a.customer.test" { |
| + t.Fatalf("after detach = %+v, want only a.customer.test", got) |
| + } |
| +} |
| + |
| +// A changed claim is re-read, and only it. This is the case a cache gets |
| +// wrong: serving a stale body because the listing said the key is still |
| +// there. |
| +func TestClaimCacheReReadsOnlyWhatChanged(t *testing.T) { |
| + dir := t.TempDir() |
| + s := &countingStore{Store: release.NewDirStore(dir)} |
| + ctx := context.Background() |
| + seedClaim(t, s, "a.customer.test", "kass") |
| + seedClaim(t, s, "b.customer.test", "kass") |
| + backdate(t, dir, "a.customer.test") |
| + backdate(t, dir, "b.customer.test") |
| + |
| + var c ClaimCache |
| + if _, err := c.List(ctx, s); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := SetClaimCapabilities(ctx, s, "b.customer.test", "acct", "kass", true, false); err != nil { |
| + t.Fatal(err) |
| + } |
| + backdate(t, dir, "b.customer.test") // settled, but at a DIFFERENT stamp |
| + s.gets = 0 |
| + got, err := c.List(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if s.gets != 1 { |
| + t.Errorf("spent %d gets, want exactly 1 — only the changed claim", s.gets) |
| + } |
| + for _, cl := range got { |
| + if cl.Host == "b.customer.test" && !cl.Wildcard { |
| + t.Error("the changed claim came back stale — the cache served a body the bucket no longer holds") |
| + } |
| + } |
| +} |
| + |
| +// THE lost update the modSettle bound exists for. S3 stamps have one-second |
| +// granularity, so a claim read and then rewritten inside the same second |
| +// carries an unchanged-looking stamp. Caching that read would serve the old |
| +// body forever, and "forever" is not an exaggeration: nothing else would |
| +// ever move the stamp. |
| +func TestClaimCacheRefusesToTrustAStampItJustSaw(t *testing.T) { |
| + dir := t.TempDir() |
| + s := &countingStore{Store: release.NewDirStore(dir)} |
| + ctx := context.Background() |
| + seedClaim(t, s, "a.customer.test", "kass") // written just now: inside modSettle |
| + |
| + var c ClaimCache |
| + if _, err := c.List(ctx, s); err != nil { |
| + t.Fatal(err) |
| + } |
| + s.gets = 0 |
| + if _, err := c.List(ctx, s); err != nil { |
| + t.Fatal(err) |
| + } |
| + if s.gets != 1 { |
| + t.Errorf("spent %d gets, want 1 — a stamp from the last two seconds must not be trusted as unchanged", s.gets) |
| + } |
| +} |
| + |
| +// A store that cannot answer ListMod — the fleet store door's client, every |
| +// test fake — degrades to the plain fold. Not to a weaker cache: a customer |
| +// fleet box is the hardest place to notice a stale claim set. |
| +func TestClaimCacheDegradesOnAStoreWithNoStamps(t *testing.T) { |
| + s := stamplessStore{release.NewDirStore(t.TempDir())} |
| + ctx := context.Background() |
| + seedClaim(t, s, "a.customer.test", "kass") |
| + var c ClaimCache |
| + got, err := c.List(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(got) != 1 { |
| + t.Fatalf("fold = %+v, want the claim", got) |
| + } |
| + // Nothing was cached, so a second fold still sees a change made between |
| + // the two — which is the whole point of degrading rather than guessing. |
| + seedClaim(t, s, "b.customer.test", "kass") |
| + if got, err := c.List(ctx, s); err != nil || len(got) != 2 { |
| + t.Fatalf("second fold = %+v, %v; want both claims", got, err) |
| + } |
| +} |
| + |
| +// stamplessStore hides DirStore's ListMod behind a plain Store, the way the |
| +// fleet door's client and the package's test fakes do. |
| +type stamplessStore struct{ inner release.Store } |
| + |
| +func (s stamplessStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + return s.inner.Get(ctx, key) |
| +} |
| +func (s stamplessStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| + return s.inner.Put(ctx, key, body, ifMatch) |
| +} |
| +func (s stamplessStore) List(ctx context.Context, prefix string) ([]string, error) { |
| + return s.inner.List(ctx, prefix) |
| +} |
| +func (s stamplessStore) ListDirs(ctx context.Context, prefix string) ([]string, error) { |
| + return s.inner.ListDirs(ctx, prefix) |
| +} |
| +func (s stamplessStore) Delete(ctx context.Context, key string) error { |
| + return s.inner.Delete(ctx, key) |
| +} |
| + |
| +// A nil cache is legal and means "no caching", so a caller with none to |
| +// offer passes nil rather than every call site growing a branch. |
| +func TestNilClaimCacheFoldsWithoutCaching(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + seedClaim(t, s, "a.customer.test", "kass") |
| + var c *ClaimCache |
| + got, err := c.List(ctx, s) |
| + if err != nil || len(got) != 1 { |
| + t.Fatalf("nil cache fold = %+v, %v", got, err) |
| + } |
| + c.Reset() // must not panic |
| +} |
| + |
| +// A transient read failure must not be remembered. Remembering it would |
| +// hide a perfectly good claim until something moved its bucket stamp — and |
| +// nothing edits a settled claim, so for that hostname "until" means never. |
| +// A decode refusal IS remembered, because the same bytes refuse the same way |
| +// every pass and re-reading them forever buys nothing. |
| +func TestClaimCacheNeverRemembersATransientReadFailure(t *testing.T) { |
| + dir := t.TempDir() |
| + inner := release.NewDirStore(dir) |
| + s := &flakyStore{Store: inner} |
| + ctx := context.Background() |
| + seedClaim(t, inner, "a.customer.test", "kass") |
| + backdate(t, dir, "a.customer.test") |
| + |
| + var c ClaimCache |
| + s.failNext = true |
| + got, err := c.List(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(got) != 0 { |
| + t.Fatalf("fold during the blip = %+v, want nothing readable", got) |
| + } |
| + // The blip is over. The claim must come back on the very next fold, |
| + // with no stamp change to prompt it. |
| + got, err = c.List(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(got) != 1 || got[0].Host != "a.customer.test" { |
| + t.Fatalf("after the blip = %+v, want the claim back — a read failure was cached", got) |
| + } |
| +} |
| + |
| +type flakyStore struct { |
| + release.Store |
| + failNext bool |
| +} |
| + |
| +func (f *flakyStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + if f.failNext { |
| + f.failNext = false |
| + return nil, "", errFlaky |
| + } |
| + return f.Store.Get(ctx, key) |
| +} |
| + |
| +func (f *flakyStore) ListMod(ctx context.Context, prefix string) ([]release.KeyMod, error) { |
| + return f.Store.(release.ModLister).ListMod(ctx, prefix) |
| +} |
| + |
| +var errFlaky = errors.New("bucket blip") |
| + |
| +// An object that will never decode is remembered, so a deployment carrying |
| +// one piece of litter under domains/ does not pay a GET for it on every |
| +// fold, on every box, forever. |
| +func TestClaimCacheRemembersAnUndecodableObject(t *testing.T) { |
| + dir := t.TempDir() |
| + s := &countingStore{Store: release.NewDirStore(dir)} |
| + ctx := context.Background() |
| + if _, err := s.Put(ctx, ClaimKey("junk.customer.test"), []byte("not json"), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + backdate(t, dir, "junk.customer.test") |
| + var c ClaimCache |
| + if got, err := c.List(ctx, s); err != nil || len(got) != 0 { |
| + t.Fatalf("fold = %+v, %v; want nothing", got, err) |
| + } |
| + s.gets = 0 |
| + if _, err := c.List(ctx, s); err != nil { |
| + t.Fatal(err) |
| + } |
| + if s.gets != 0 { |
| + t.Errorf("spent %d gets re-reading an object that will never decode", s.gets) |
| + } |
| +} |
| + |
| +// The entry map is replaced whole rather than merged, so a deployment that |
| +// churns through hostnames does not accumulate an entry per hostname it has |
| +// ever held. Reaching into the field is white-box on purpose: there is no |
| +// behaviour to observe here, only unbounded growth, and unbounded growth |
| +// with no test is how a leak reaches production. |
| +func TestClaimCacheDoesNotGrowWithChurnedHostnames(t *testing.T) { |
| + dir := t.TempDir() |
| + s := release.NewDirStore(dir) |
| + ctx := context.Background() |
| + var c ClaimCache |
| + for i, host := range []string{"one.customer.test", "two.customer.test", "three.customer.test"} { |
| + seedClaim(t, s, host, "kass") |
| + backdate(t, dir, host) |
| + if _, err := c.List(ctx, s); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := DetachClaim(ctx, s, host, "acct", "kass"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := c.List(ctx, s); err != nil { |
| + t.Fatal(err) |
| + } |
| + c.mu.Lock() |
| + n := len(c.entries) |
| + c.mu.Unlock() |
| + if n != 0 { |
| + t.Fatalf("after churning %d hostname(s) the cache holds %d entries, want 0", i+1, n) |
| + } |
| + } |
| +} |
diff --git a/internal/domains/claims.go b/internal/domains/claims.go| index 7cdb7dacc..ec8c8533e 100644 |
| --- a/internal/domains/claims.go |
| +++ b/internal/domains/claims.go |
| @@ -179,25 +179,46 @@ func ListClaims(ctx context.Context, s release.Store) ([]Claim, error) { |
| } |
| var claims []Claim |
| for _, key := range keys { |
| - raw, _, err := s.Get(ctx, key) |
| - if err != nil { |
| - log.Printf("domains: claim %s: %v", key, err) |
| - continue |
| - } |
| - var c Claim |
| - if err := json.Unmarshal(raw, &c); err != nil { |
| - log.Printf("domains: claim %s: unparseable: %v", key, err) |
| - continue |
| + if c, ok, _ := readClaim(ctx, s, key); ok { |
| + claims = append(claims, c) |
| } |
| - if c.Schema > 1 { |
| - log.Printf("domains: claim %s: schema %d is newer than this binary understands, skipping", key, c.Schema) |
| - continue |
| - } |
| - claims = append(claims, c) |
| } |
| return claims, nil |
| } |
| +// readClaim reads and decodes one claim object. Every refusal is a skip and |
| +// never an error to the caller: one unparseable or too-new object under the |
| +// prefix must not take down the whole fold. |
| +// |
| +// The two failure kinds are told apart, and the distinction is load-bearing |
| +// for ClaimCache. A DECODE refusal (unparseable, or a schema this binary |
| +// does not understand) is deterministic — the same bytes will refuse the |
| +// same way next pass — so it is safe to remember. A READ failure is a |
| +// network blip and is not: remembering it would hide a perfectly good claim |
| +// until something unrelated moved its bucket stamp, which for a claim that |
| +// nobody edits is indistinguishable from forever. So a read failure reports |
| +// firm=false and the cache declines to keep it. |
| +// |
| +// Shared with ClaimCache so the two folds cannot drift into different ideas |
| +// of what counts as a claim — which would show up as a hostname that appears |
| +// on a cached read and vanishes on an uncached one. |
| +func readClaim(ctx context.Context, s release.Store, key string) (c Claim, ok, firm bool) { |
| + raw, _, err := s.Get(ctx, key) |
| + if err != nil { |
| + log.Printf("domains: claim %s: %v", key, err) |
| + return Claim{}, false, false |
| + } |
| + if err := json.Unmarshal(raw, &c); err != nil { |
| + log.Printf("domains: claim %s: unparseable: %v", key, err) |
| + return Claim{}, false, true |
| + } |
| + if c.Schema > 1 { |
| + log.Printf("domains: claim %s: schema %d is newer than this binary understands, skipping", key, c.Schema) |
| + return Claim{}, false, true |
| + } |
| + return c, true, true |
| +} |
| + |
| // hostLabelRe mirrors provision.labelRe (which itself mirrors |
| // domains.accountNameRe — see that var's comment) — pinned HERE rather than |
| // imported from internal/provision, because internal/provision/reconcile.go |
diff --git a/internal/host/agent.go b/internal/host/agent.go| index 13bb1580c..664ba92e6 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -1459,7 +1459,7 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service, sd *schedu |
| // cache would need its own staleness rule, and a STALE route set |
| // is the one thing a security guard must not have. |
| deploymentRoutes := edgesfile.DeploymentRoutes(ctx, store, time.Now()) |
| - if err := domains.ReconcileAliases(ctx, store, reg, time.Now, deploymentRoutes); err != nil { |
| + if err := domains.ReconcileAliases(ctx, store, reg, time.Now, deploymentRoutes, &aliasClaimCache); err != nil { |
| log.Printf("agent: fast converge alias pass: %v", err) |
| } |
| if err := release.Pass(ctx, reg, store, pub, optDir, restart, opts...); err != nil { |
| @@ -1917,7 +1917,7 @@ func instancesLoop(reg *registry.Registry) { |
| log.Printf("agent: instances pass: %v", err) |
| } |
| deploymentRoutes := edgesfile.DeploymentRoutes(ctx, store, time.Now()) |
| - if err := domains.ReconcileAliases(ctx, store, reg, time.Now, deploymentRoutes); err != nil { |
| + if err := domains.ReconcileAliases(ctx, store, reg, time.Now, deploymentRoutes, &aliasClaimCache); err != nil { |
| log.Printf("agent: instances alias pass: %v", err) |
| } |
| if !isFleetBox { |
diff --git a/internal/host/edgesfile.go b/internal/host/edgesfile.go| index d57bbd1aa..0eed5d7c0 100644 |
| --- a/internal/host/edgesfile.go |
| +++ b/internal/host/edgesfile.go |
| @@ -14,6 +14,8 @@ import ( |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| + |
| + "github.com/carlosframework/platform/internal/domains" |
| ) |
| // The multi-region ingress spec (2026-08-20, Part 3, closes the design half |
| @@ -384,3 +386,17 @@ func edgesFileLoop(reg *registry.Registry) { |
| } |
| } |
| } |
| + |
| +// aliasClaimCache is the claim fold both alias legs share (the fast-converge |
| +// leg in fullPass and the five-minute instances pass). Process-lifetime and |
| +// package-level for the same reason withheldAliasSaid above is: there is one |
| +// agent per process, and both legs are asking about the same deployment's |
| +// claims, so two caches would double the cold reads and halve the hit rate |
| +// for nothing. |
| +// |
| +// The fast leg is why this exists at all. It runs on every serial bump — a |
| +// seconds cadence — and ListClaims is one List plus a GET per claim in the |
| +// WHOLE deployment. At the hundreds of custom domains titogo is heading for, |
| +// that is a steady drip of bucket reads on every box, sitting ahead of the |
| +// adopt leg inside a two-second convergence promise. See domains.ClaimCache. |
| +var aliasClaimCache domains.ClaimCache |