amadan
Merge pull request #366 from carlosframework/cert-sweep
domains: the Settings page's state, the claim-fold cache, and the cert sweep (#299 §3f, #173, #243)
17 files changed,
+1363
−62
docs/superpowers/specs/2026-08-27-managed-custom-domains-design.md+32 −5internal/console/apidomains.go+1 −1internal/console/appsettings.go+32 −7internal/console/appsettings_test.go+113 −0internal/console/server.go+7 −2internal/console/templates/app_settings.html+8 −6internal/console/view.go+46 −0internal/domains/aliases.go+7 −2internal/domains/aliases_test.go+20 −20internal/domains/claimcache.go+136 −0internal/domains/claimcache_test.go+374 −0internal/domains/claims.go+35 −14internal/edge/certsweep.go+194 −0internal/edge/certsweep_test.go+321 −0internal/edge/certwarm.go+19 −3internal/host/agent.go+2 −2internal/host/edgesfile.go+16 −0
diff --git a/docs/superpowers/specs/2026-08-27-managed-custom-domains-design.md b/docs/superpowers/specs/2026-08-27-managed-custom-domains-design.md| index 0870d9ffa..9d1020a2e 100644 |
| --- a/docs/superpowers/specs/2026-08-27-managed-custom-domains-design.md |
| +++ b/docs/superpowers/specs/2026-08-27-managed-custom-domains-design.md |
| @@ -695,11 +695,38 @@ alternative to `issuing-cert`. The vocabulary honesty test now produces it |
| from an input the platform can actually produce, which is the same bar every |
| other state clears. |
| -§3f remains open. Consequence stated plainly: `control/certs/` entries for |
| -detached hosts linger until something else removes them. That is not a |
| -correctness or safety problem — §3f's whole ruling was that the sweep must |
| -key on EXPIRY rather than detachment, so nothing is being deleted wrongly by |
| -not running it. |
| +**§3f is built (2026-08-29)**, in `internal/edge/certsweep.go`, and it took |
| +#243's spent-token litter with it — one issuer-side pass over |
| +`control/certs/`, two predicates, because the two kinds of litter are not the |
| +same kind of thing: |
| + |
| +- **Spent challenge tokens** (#243) go on AGE alone. A validation's whole |
| + life is seconds and autocert's order context is five minutes, so the |
| + one-hour TTL is generous by two orders of magnitude — deliberately, since |
| + deleting a token out from under a live validation burns the order. |
| +- **Certificates** go only on their own leaf's expiry, plus a month. §3f's |
| + ruling is unchanged and is the reason the file opens with it: a sweep keyed |
| + "no claim exists for this host" would delete every platform host's cert, |
| + every wildcard PEM and the account keys, and a registry lag could take a |
| + live host down fleet-wide through the mirror. |
| + |
| +Three bounds the spec did not name: |
| + |
| +- **A parse floor.** Nothing younger than sixty days is read at all. autocert |
| + renews at ~30 days remaining on a 90-day certificate, so a healthy entry's |
| + object stamp never gets that old — which makes the steady-state cost one |
| + listing and zero reads, instead of the per-object N+1 this would otherwise |
| + be. An entry that IS that old is one nothing is renewing, which is exactly |
| + the population worth reading. |
| +- **Anything unreadable is left alone.** A sweep that deletes what it cannot |
| + parse is a sweep that deletes tomorrow's format. |
| +- **Fifty deletions per pass.** A bug in the predicate removes at most fifty |
| + objects before somebody reads the log. |
| + |
| +The gate check moved above the empty-warm-set return in `pass()`: token |
| +litter and dead certs exist in a deployment with no custom domains at all, |
| +and the old ordering would have meant such a deployment never swept. A |
| +non-issuer still sweeps nothing. |
| ### The shared reader is a READER, not the wire-shape owner |
diff --git a/internal/console/apidomains.go b/internal/console/apidomains.go| index 5d78f5b13..6991285a0 100644 |
| --- a/internal/console/apidomains.go |
| +++ b/internal/console/apidomains.go |
| @@ -677,7 +677,7 @@ func (s *Server) handleAPIDomainList(w http.ResponseWriter, r *http.Request) { |
| // cross-account uniqueness gate), so the tenant boundary is this filter |
| // and nothing else: every row that reaches the answer matched BOTH this |
| // account's sqid and this app. |
| - claims, err := domains.ListClaims(ctx, s.cfg.Releases) |
| + claims, err := s.claims.List(ctx, s.cfg.Releases) |
| if err != nil { |
| log.Printf("console: api domains list %s/%s: %v", acct.Sqid, app, err) |
| jsonError(w, http.StatusBadGateway, "could not list the hostnames") |
diff --git a/internal/console/appsettings.go b/internal/console/appsettings.go| index a8f2ae177..88a16d976 100644 |
| --- a/internal/console/appsettings.go |
| +++ b/internal/console/appsettings.go |
| @@ -38,7 +38,6 @@ import ( |
| "strings" |
| "time" |
| - "github.com/carlosframework/platform/internal/domains" |
| "github.com/carlosframework/platform/internal/mail" |
| "github.com/carlosframework/platform/internal/objstore" |
| "github.com/carlosframework/platform/internal/release" |
| @@ -406,10 +405,23 @@ func (s *Server) mailCard(ctx context.Context, acct *Account, app string) *MailC |
| } |
| // settingsDomains is the Settings page's own Domains & TLS rows: routed |
| -// (the AppView's own DomainView slice, untouched) plus a placeholder row |
| -// for any of this app's domain claims that routed has no entry for yet. |
| -// Best-effort like the rest of this file — a claim-list read failure is |
| -// logged and the page falls back to routed alone, never a hard error. |
| +// (the AppView's own DomainView slice) plus a placeholder row for any of |
| +// this app's domain claims that routed has no entry for yet — and, for every |
| +// claim-backed row, the derived lifecycle state. |
| +// |
| +// The state is the point. Until it was threaded here the Settings page was |
| +// the last first-party surface still telling the pre-#299 story: it read |
| +// THIS box's own registry copy of a route and printed a DNS word and a TLS |
| +// word, while `carlos domains list` — reading the fleet-wide sweep — could |
| +// be saying something else about the same hostname at the same moment. Both |
| +// now come from domainState(), over the same inputs. |
| +// |
| +// Best-effort like the rest of this file. A claim-list read failure is |
| +// logged and the page falls back to routed alone; a readings or warm-record |
| +// failure leaves the state empty and the row renders exactly as it did |
| +// before. Neither is ever a hard error — a settings page that 500s because a |
| +// telemetry prefix was briefly unreadable would be a worse answer than one |
| +// that shows a little less. |
| func (s *Server) settingsDomains(ctx context.Context, acct *Account, app string, routed []DomainView) []DomainView { |
| out := append([]DomainView(nil), routed...) |
| if s.cfg.Releases == nil { |
| @@ -419,20 +431,33 @@ func (s *Server) settingsDomains(ctx context.Context, acct *Account, app string, |
| for i, d := range out { |
| have[strings.ToLower(d.Domain)] = i |
| } |
| - claims, err := domains.ListClaims(ctx, s.cfg.Releases) |
| + claims, err := s.claims.List(ctx, s.cfg.Releases) |
| if err != nil { |
| log.Printf("console: settings claims %s/%s: %v", acct.Sqid, app, err) |
| return out |
| } |
| + // Folded ONCE for the page, not once per row: readings is every live |
| + // box's alias stamps, warm is one object per hostname currently failing |
| + // to get a certificate (usually none at all). |
| + now := time.Now() |
| + readings := aliasReadings(ctx, s.cfg.Releases, now) |
| + warm := warmRecords(ctx, s.cfg.Releases) |
| + state := func(host string) (st, reason, detail string) { |
| + reading, seen := readings[host] |
| + return domainState(reading, seen, warm[host], now) |
| + } |
| for _, c := range claims { |
| if c.Account != acct.Sqid || c.App != app { |
| continue |
| } |
| if i, ok := have[strings.ToLower(c.Host)]; ok { |
| out[i].Claimed = true |
| + out[i].State, out[i].Reason, out[i].Detail = state(out[i].Domain) |
| continue |
| } |
| - out = append(out, claimDomainView(c.Host, s.dnsTargets())) |
| + dv := claimDomainView(c.Host, s.dnsTargets()) |
| + dv.State, dv.Reason, dv.Detail = state(c.Host) |
| + out = append(out, dv) |
| } |
| return out |
| } |
diff --git a/internal/console/appsettings_test.go b/internal/console/appsettings_test.go| index 5c01b0b21..f4b6edde1 100644 |
| --- a/internal/console/appsettings_test.go |
| +++ b/internal/console/appsettings_test.go |
| @@ -12,6 +12,9 @@ package console |
| import ( |
| "context" |
| "errors" |
| + "github.com/carlosframework/platform/internal/certwarm" |
| + "github.com/carlosframework/platform/internal/domains" |
| + "github.com/carlosframework/platform/internal/status" |
| "net/http" |
| "net/url" |
| "strings" |
| @@ -1439,3 +1442,113 @@ func TestSettingsPageEmailUnavailableOnAConsoleWithNoProvisioner(t *testing.T) { |
| t.Fatalf("a provisioner-less console still suggests a command that would 404:\n%s", body) |
| } |
| } |
| + |
| +// The Settings page was the last first-party surface still telling the |
| +// pre-#299 story: it read THIS box's registry copy and printed a DNS word, |
| +// while `carlos domains list` read the fleet-wide sweep and could be saying |
| +// something else about the same hostname at the same moment. Both must now |
| +// come from domainState over the same inputs. |
| +func TestSettingsDomainsCarryTheSameStateTheAPIDoorDerives(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + now := time.Now() |
| + acct, err := fx.st.AccountBySqid(fx.sqid) |
| + if err != nil || acct == nil { |
| + t.Fatalf("fixture account %s: %v", fx.sqid, err) |
| + } |
| + if _, _, err := domains.AttachClaim(ctx, fx.rel, domains.Claim{ |
| + Host: "www.customer.test", Account: fx.sqid, App: "kass", |
| + }); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := status.WriteBox(ctx, fx.rel, status.Box{ |
| + GeneratedAt: now, Label: "ada-reporting", Region: "eu-west-1", |
| + Metrics: &status.BoxMetrics{Aliases: map[string]status.AliasReading{ |
| + "www.customer.test": {Route: "kass.carlosapp.dev", DNSStatus: "elsewhere", DNSCheckedAt: now.Unix()}, |
| + }}, |
| + }); err != nil { |
| + t.Fatal(err) |
| + } |
| + got := fx.srv.settingsDomains(ctx, acct, "kass", nil) |
| + if len(got) != 1 { |
| + t.Fatalf("rows = %+v", got) |
| + } |
| + wantState, wantReason, wantDetail := domainState( |
| + status.AliasReading{Route: "kass.carlosapp.dev", DNSStatus: "elsewhere", DNSCheckedAt: now.Unix()}, |
| + true, nil, now) |
| + if got[0].State != wantState || got[0].Reason != wantReason || got[0].Detail != wantDetail { |
| + t.Errorf("row = state %q reason %q detail %q; want %q / %q / %q — the page and the CLI must derive one verdict", |
| + got[0].State, got[0].Reason, got[0].Detail, wantState, wantReason, wantDetail) |
| + } |
| + if got[0].Status() != wantState { |
| + t.Errorf("Status() = %q, want the derived state %q", got[0].Status(), wantState) |
| + } |
| +} |
| + |
| +// A failing certificate reaches the page, through the same warm-outcome |
| +// record the API doors read (#299 §3e). This is the surface a member is most |
| +// likely to be looking at when it happens. |
| +func TestSettingsDomainsShowAFailedCertificate(t *testing.T) { |
| + fx := newConsoleFixture(t) |
| + ctx := context.Background() |
| + now := time.Now() |
| + acct, err := fx.st.AccountBySqid(fx.sqid) |
| + if err != nil || acct == nil { |
| + t.Fatalf("fixture account %s: %v", fx.sqid, err) |
| + } |
| + if _, _, err := domains.AttachClaim(ctx, fx.rel, domains.Claim{ |
| + Host: "broken.customer.test", Account: fx.sqid, App: "kass", |
| + }); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := status.WriteBox(ctx, fx.rel, status.Box{ |
| + GeneratedAt: now, Label: "ada-reporting", Region: "eu-west-1", |
| + Metrics: &status.BoxMetrics{Aliases: map[string]status.AliasReading{ |
| + "broken.customer.test": {Route: "kass.carlosapp.dev", DNSStatus: "pointing", DNSCheckedAt: now.Unix()}, |
| + }}, |
| + }); err != nil { |
| + t.Fatal(err) |
| + } |
| + body, err := certwarm.Record{Host: "broken.customer.test", Attempts: domainFailedAttempts, |
| + Class: certwarm.ClassOther, NextAttemptAt: now.Add(time.Hour).Unix()}.Encode(now) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := fx.rel.Put(ctx, certwarm.Key("broken.customer.test"), body, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + got := fx.srv.settingsDomains(ctx, acct, "kass", nil) |
| + if len(got) != 1 || got[0].State != StateFailed { |
| + t.Fatalf("rows = %+v, want one row in state %q", got, StateFailed) |
| + } |
| + // And it EXPLAINS. A one-word "failed" with no sentence under it is the |
| + // worst version of this row. |
| + if got[0].Explain() == "" { |
| + t.Error("a failed certificate row carries no explanation") |
| + } |
| +} |
| + |
| +// Explain is what keeps the page readable: a working domain gets no |
| +// paragraph, because a sentence repeated under every healthy row trains |
| +// people to skip the one under the broken row. |
| +func TestDomainViewExplainStaysQuietWhenThereIsNothingToSay(t *testing.T) { |
| + for _, tc := range []struct { |
| + name string |
| + d DomainView |
| + want string |
| + }{ |
| + {"live says nothing", DomainView{State: StateLive, Detail: "served over HTTPS"}, ""}, |
| + {"a problem says its detail", DomainView{State: StateMisdirected, Detail: "points elsewhere"}, "points elsewhere"}, |
| + {"a problem with records says both", |
| + DomainView{State: StatePendingDNS, Detail: "no record yet", Guidance: "Point an A record."}, |
| + "no record yet Point an A record."}, |
| + // A row the shared view builder produced, with no fleet fold behind |
| + // it, must render exactly as it always has. |
| + {"no state falls back to guidance", DomainView{Guidance: "Point an A record."}, "Point an A record."}, |
| + {"no state and no guidance", DomainView{}, ""}, |
| + } { |
| + if got := tc.d.Explain(); got != tc.want { |
| + t.Errorf("%s: Explain() = %q, want %q", tc.name, got, tc.want) |
| + } |
| + } |
| +} |
diff --git a/internal/console/server.go b/internal/console/server.go| index 5ec66f115..1acdcf8bd 100644 |
| --- a/internal/console/server.go |
| +++ b/internal/console/server.go |
| @@ -588,8 +588,13 @@ type Config struct { |
| // Server is the console's http.Handler. |
| type Server struct { |
| - cfg Config |
| - store *Store |
| + cfg Config |
| + store *Store |
| + // claims folds the deployment's domain claims for the read paths — the |
| + // Settings render and the domains list door — reading only the claims |
| + // whose bucket stamp moved since the last fold (#173). Process-lifetime |
| + // and safe for concurrent handlers; see domains.ClaimCache. |
| + claims domains.ClaimCache |
| km *Keymail |
| mux *http.ServeMux |
| instances *instanceFetcher |
diff --git a/internal/console/templates/app_settings.html b/internal/console/templates/app_settings.html| index 75f72e3e8..5dafd1603 100644 |
| --- a/internal/console/templates/app_settings.html |
| +++ b/internal/console/templates/app_settings.html |
| @@ -37,16 +37,16 @@ bucket keys and stays.</p> |
| {{if .Settings.Domains}} |
| <div class="tablewrap"> |
| <table> |
| - <tr><th>Domain</th><th>Type</th><th>DNS</th><th>TLS</th></tr> |
| + <tr><th>Domain</th><th>Type</th><th>Status</th><th>Certificate</th></tr> |
| {{range $i, $d := .Settings.Domains}} |
| <tr> |
| <td><code>{{$d.Domain}}</code></td> |
| <td>{{$d.Type}}</td> |
| - <td>{{if $d.DNS}}{{$d.DNS}}{{else}}<span title="unknown — not checked yet">—</span>{{end}}</td> |
| + <td>{{with $d.Status}}{{.}}{{else}}<span title="unknown — not checked yet">—</span>{{end}}</td> |
| <td>{{if $d.TLS}}{{$d.TLS}}{{else}}<span title="unknown">—</span>{{end}}</td> |
| </tr> |
| - {{if $d.Guidance}} |
| - <tr><td colspan="4" class="small">{{$d.Guidance}}</td></tr> |
| + {{with $d.Explain}} |
| + <tr><td colspan="4" class="small">{{.}}</td></tr> |
| {{end}} |
| {{if and $.IsOwner $d.Claimed}} |
| <tr><td colspan="4" class="small"> |
| @@ -78,8 +78,10 @@ at its platform domain; attaching adds addresses, it never removes one.</p>{{end |
| {{end}} |
| <button type="submit" class="btn-sm">Attach domain</button> |
| </form> |
| -<p class="small">Point the domain's A record at this deployment's edge; |
| -the certificate is minted on the first request and renews on its own.</p> |
| +<p class="small">Point the domain's DNS at this deployment — once it is |
| +attached, the row above says exactly what to create. The platform obtains the |
| +certificate itself and renews it, so nothing has to visit the domain |
| +first.</p> |
| {{end}} |
| {{if and .Settings .Settings.MismatchTarget}} |
diff --git a/internal/console/view.go b/internal/console/view.go| index c0c2a9926..ec65561ed 100644 |
| --- a/internal/console/view.go |
| +++ b/internal/console/view.go |
| @@ -128,6 +128,17 @@ func (v AppView) AccountMismatch() bool { return len(v.Mismatched) > 0 } |
| type DomainView struct { |
| Domain, Type, DNS, TLS string |
| Guidance string |
| + // State, Reason and Detail are the derived lifecycle words (#299 §2a) — |
| + // the SAME three domainState() gives `carlos domains list` and the API |
| + // doors, from the same fleet-wide readings, so the console page and the |
| + // CLI can never tell a member two stories about one hostname. That was |
| + // the whole complaint #299 opened with. |
| + // |
| + // Empty on a row that has not been through settingsDomains: the shared |
| + // view builder (domainView, below) folds one box's own registry copy and |
| + // has no fleet fold to derive a state from. Readers must treat "" as |
| + // "this build did not derive one here", never as a verdict. |
| + State, Reason, Detail string |
| // Claimed is Settings-only (settingsDomains, appsettings.go): true when |
| // this row corresponds to a domain claim this app holds — routed or not |
| // — which is what the Settings page uses to decide whether a detach |
| @@ -136,6 +147,41 @@ type DomainView struct { |
| Claimed bool |
| } |
| +// Status is the one word the Domains table shows: the derived lifecycle |
| +// state where there is one, and otherwise the raw DNS word this box |
| +// stamped. Never both — a Status column and a DNS column side by side would |
| +// be two answers to one question, which is what #299 was opened about. |
| +func (d DomainView) Status() string { |
| + if d.State != "" { |
| + return d.State |
| + } |
| + return d.DNS |
| +} |
| + |
| +// Explain is the sentence under a Domains row, or "" for no sentence. |
| +// |
| +// A healthy row gets none: "live" beside a current renewal date says |
| +// everything, and a paragraph repeating it under every working domain is |
| +// noise that trains people to skip the paragraph under the broken one. |
| +// Everything else gets the state's own detail — the same sentence the CLI |
| +// prints, verbatim — followed by the DNS instructions when there are any. |
| +// |
| +// A row with no state at all keeps exactly today's behaviour: guidance |
| +// alone. That is the platform-domain rows and anything the shared view |
| +// builder produced without a fleet fold. |
| +func (d DomainView) Explain() string { |
| + if d.State == "" { |
| + return d.Guidance |
| + } |
| + if d.State == StateLive || d.Detail == "" { |
| + return "" |
| + } |
| + if d.Guidance == "" { |
| + return d.Detail |
| + } |
| + return d.Detail + " " + d.Guidance |
| +} |
| + |
| // DNSTargets is where this deployment tells a member to point a custom |
| // domain. Two facts, threaded together because the guidance is one |
| // sentence and needs both: the edge addresses an A record names, and — |
diff --git a/internal/domains/aliases.go b/internal/domains/aliases.go| index 77b23ce7b..21530eb4c 100644 |
| --- a/internal/domains/aliases.go |
| +++ b/internal/domains/aliases.go |
| @@ -40,8 +40,13 @@ const maxWildcardDomains = 32 |
| // no instance record to read. nil means "nothing known" and is PERMISSIVE by |
| // contract — a bucket the fold could not read must never break ordinary |
| // attach. |
| -func ReconcileAliases(ctx context.Context, s release.Store, reg *registry.Registry, now func() time.Time, deploymentRoutes map[string]bool) error { |
| - claims, err := ListClaims(ctx, s) |
| +// cache, when non-nil, folds the claims through a ClaimCache instead of |
| +// re-reading every claim body. This pass runs on the fast-converge leg — a |
| +// seconds cadence, on every serial bump — so the difference is one bucket |
| +// call against one plus a GET per claim in the deployment, every few |
| +// seconds, on every box. nil is legal and means today's uncached fold. |
| +func ReconcileAliases(ctx context.Context, s release.Store, reg *registry.Registry, now func() time.Time, deploymentRoutes map[string]bool, cache *ClaimCache) error { |
| + claims, err := cache.List(ctx, s) |
| if err != nil { |
| return err |
| } |
diff --git a/internal/domains/aliases_test.go b/internal/domains/aliases_test.go| index 2507395f8..d51f64c4c 100644 |
| --- a/internal/domains/aliases_test.go |
| +++ b/internal/domains/aliases_test.go |
| @@ -77,7 +77,7 @@ func TestReconcileAliasesMaterializes(t *testing.T) { |
| putClaim(t, s, Claim{ |
| Host: "tickets.example.com", Account: "bab", App: "kass", Route: "canary/live", |
| }) |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -103,7 +103,7 @@ func TestReconcileAliasesOwnershipGate(t *testing.T) { |
| putClaim(t, s, Claim{ |
| Host: "tickets.example.com", Account: "bab", App: "somethingelse", Route: "canary/live", |
| }) |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -129,7 +129,7 @@ func TestReconcileAliasesOwnershipGateAccountMismatch(t *testing.T) { |
| putClaim(t, s, Claim{ |
| Host: "tickets.example.com", Account: "bab", App: "kass", Route: "canary/live", |
| }) |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -150,7 +150,7 @@ func TestReconcileAliasesRoutesWin(t *testing.T) { |
| putClaim(t, s, Claim{ |
| Host: "tickets.example.com", Account: "bab", App: "kass", Route: "canary/live", |
| }) |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -168,7 +168,7 @@ func TestReconcileAliasesMissingRouteSkips(t *testing.T) { |
| putClaim(t, s, Claim{ |
| Host: "tickets.example.com", Account: "bab", App: "kass", Route: "canary/live", |
| }) |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -192,7 +192,7 @@ func TestReconcileAliasesRemovesOrphans(t *testing.T) { |
| t.Fatal(err) |
| } |
| // No claim backs orphan.example.com. |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -213,7 +213,7 @@ func TestReconcileAliasesRouteless(t *testing.T) { |
| putClaim(t, s, Claim{ |
| Host: "tickets.example.com", Account: "bab", App: "kass", |
| }) |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -233,7 +233,7 @@ func TestReconcileAliasesIdempotent(t *testing.T) { |
| putClaim(t, s, Claim{ |
| Host: "tickets.example.com", Account: "bab", App: "kass", Route: "canary/live", |
| }) |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| first, err := reg.ListAliases() |
| @@ -247,7 +247,7 @@ func TestReconcileAliasesIdempotent(t *testing.T) { |
| // Second pass, later clock: CreatedAt must be preserved, not bumped. |
| later := func() time.Time { return time.Unix(1755000000+3600, 0) } |
| - if err := ReconcileAliases(context.Background(), s, reg, later, nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, later, nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| second, err := reg.ListAliases() |
| @@ -284,7 +284,7 @@ func TestReconcileWildcardCreatedAtStable(t *testing.T) { |
| Route: "woodstar-relay.bab.oncarlos.com", CreatedAt: "2026-08-13T00:00:00Z", |
| Wildcard: true, Catchall: true}) |
| - if err := ReconcileAliases(context.Background(), s, reg, func() time.Time { return time.Unix(1000, 0) }, nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, func() time.Time { return time.Unix(1000, 0) }, nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| first, err := reg.ListWildcardDomains() |
| @@ -300,7 +300,7 @@ func TestReconcileWildcardCreatedAtStable(t *testing.T) { |
| // not bumped — a bump here would move Fingerprint's wildcard_domains |
| // section every poll and force a reload→refresh loop with nothing |
| // actually changed. |
| - if err := ReconcileAliases(context.Background(), s, reg, func() time.Time { return time.Unix(1000+3600, 0) }, nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, func() time.Time { return time.Unix(1000+3600, 0) }, nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| second, err := reg.ListWildcardDomains() |
| @@ -335,7 +335,7 @@ func TestReconcileDerivesWildcardState(t *testing.T) { |
| putClaim(t, s, Claim{Host: "bad host.example", Account: "bab", App: "woodstar-relay", |
| Route: "woodstar-relay.bab.oncarlos.com", CreatedAt: "2026-08-13T00:00:00Z", Wildcard: true, Catchall: true}) |
| - if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(1000, 0) }, nil); err != nil { |
| + if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(1000, 0) }, nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| @@ -359,7 +359,7 @@ func TestReconcileDerivesWildcardState(t *testing.T) { |
| // Claim removed => derived state removed (pass-owned). |
| deleteClaim(t, s, "woodstar.app") |
| - if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(2000, 0) }, nil); err != nil { |
| + if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(2000, 0) }, nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| wds, _ = reg.ListWildcardDomains() |
| @@ -392,7 +392,7 @@ func TestReconcileWildcardHostCollisionMismatchedTenant(t *testing.T) { |
| Route: "primary.bab.oncarlos.com", CreatedAt: "2026-08-13T00:00:00Z", |
| Wildcard: true, Catchall: true}) |
| - if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(1000, 0) }, nil); err != nil { |
| + if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(1000, 0) }, nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| wds, err := reg.ListWildcardDomains() |
| @@ -427,7 +427,7 @@ func TestReconcileWildcardCap(t *testing.T) { |
| Route: "primary.bab.oncarlos.com", CreatedAt: "2026-08-13T00:00:00Z", Wildcard: true}) |
| } |
| - if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(1000, 0) }, nil); err != nil { |
| + if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(1000, 0) }, nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| wds, err := reg.ListWildcardDomains() |
| @@ -454,7 +454,7 @@ func TestReconcileWildcardEmptyRouteClaim(t *testing.T) { |
| putClaim(t, s, Claim{Host: "story.example", Account: "bab", App: "story", |
| CreatedAt: "2026-08-13T00:00:00Z", Wildcard: true, Catchall: true}) |
| - if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(1000, 0) }, nil); err != nil { |
| + if err := ReconcileAliases(ctx, s, reg, func() time.Time { return time.Unix(1000, 0) }, nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| wds, _ := reg.ListWildcardDomains() |
| @@ -491,7 +491,7 @@ func TestReconcileAliasesRefusesAHostAnotherBoxServesAsARoute(t *testing.T) { |
| // go.tito.io is a real, non-alias route on some OTHER box in the fleet. |
| deploymentRoutes := map[string]bool{"go.tito.io": true} |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), deploymentRoutes); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), deploymentRoutes, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -524,7 +524,7 @@ func TestReconcileAliasesRemovesAnAliasThatNowCollidesWithADeploymentRoute(t *te |
| deploymentRoutes := map[string]bool{"go.tito.io": true} |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), deploymentRoutes); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), deploymentRoutes, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -553,7 +553,7 @@ func TestReconcileAliasesStillMaterializesWhenTheRouteSetIsUnknown(t *testing.T) |
| Host: "shop.customer.com", Account: "bab", App: "kass", Route: "canary/live", |
| }) |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), nil, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
| @@ -593,7 +593,7 @@ func TestReconcileAliasesStillDerivesTheCatchAllForAFirstRouteClaim(t *testing.T |
| // The claimant's own box publishes woodstar.app as a REAL route. |
| deploymentRoutes := map[string]bool{"woodstar.app": true} |
| - if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), deploymentRoutes); err != nil { |
| + if err := ReconcileAliases(context.Background(), s, reg, aliasNow(), deploymentRoutes, nil); err != nil { |
| t.Fatal(err) |
| } |
| aliases, err := reg.ListAliases() |
diff --git a/internal/domains/claimcache.go b/internal/domains/claimcache.go| new file mode 100644 |
| index 000000000..b64c5f882 |
| --- /dev/null |
| +++ b/internal/domains/claimcache.go |
| @@ -0,0 +1,136 @@ |
| +package domains |
| + |
| +import ( |
| + "context" |
| + "sync" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// ClaimCache makes the claim fold cost one bucket call in the steady state |
| +// instead of one plus a GET per claim (#173, #165). |
| +// |
| +// The problem it solves. ListClaims lists domains/ and then reads every |
| +// object under it, and three things do that repeatedly: every app Settings |
| +// render, every `carlos domains list`, and every box's ReconcileAliases pass |
| +// — the last one on a five-minute floor, ahead of the adopt leg inside a |
| +// two-second convergence promise. At tens of claims that is invisible. At |
| +// the hundreds titogo's custom domains are heading for it is a slow page and |
| +// a steady drip of bucket reads on every box in the fleet, forever. |
| +// |
| +// The fix needs no new object and no new writer, because the listing already |
| +// carries the answer: S3 returns LastModified with every ListObjectsV2 entry |
| +// and DirStore has the file mtime, both of which release.ModLister already |
| +// surfaces and List throws away. A claim whose last-modified time has not |
| +// moved has not changed, so its body does not need re-reading. The steady |
| +// state is therefore ONE ListMod and zero GETs, and the first pass after a |
| +// claim changes reads exactly the claims that changed. |
| +// |
| +// This is a cache with no TTL and no invalidation call, which is the point: |
| +// it is not guessing about staleness, it is being told. Two exact rules do |
| +// all the work — an entry is reused only when its stamp is unchanged, and an |
| +// entry whose key has left the listing is dropped, which is the detach path. |
| +// |
| +// A store that cannot answer ListMod (the fleet store door's client, every |
| +// test fake) degrades to a plain ListClaims with no caching at all. That is |
| +// deliberate: a fallback that cached on time instead would be a different, |
| +// weaker contract wearing the same name, and the one place it would be used |
| +// is the one place — a customer fleet box — where a stale claim set is |
| +// hardest to notice. |
| +// |
| +// Safe for concurrent use. The zero value is ready, and a nil *ClaimCache |
| +// is legal and means "no caching" — so a caller that has no cache to offer |
| +// passes nil rather than every call site growing a branch. |
| +type ClaimCache struct { |
| + mu sync.Mutex |
| + entries map[string]cachedClaim |
| +} |
| + |
| +type cachedClaim struct { |
| + mod time.Time |
| + claim Claim |
| + // skipped records that this key was listed but produced no claim for a |
| + // DETERMINISTIC reason — unparseable, or a schema newer than this |
| + // binary. Cached like a hit so a permanently undecodable object does not |
| + // earn a GET on every pass forever. A key whose READ failed is never |
| + // cached at all: see readClaim's firm return. |
| + skipped bool |
| +} |
| + |
| +// modSettle is how recently an object may have been written before its |
| +// stamp stops being trustworthy as an unchanged-marker. |
| +// |
| +// S3 last-modified has one-second granularity. Without this bound there is a |
| +// real lost update: read a claim at T and cache stamp T, have it rewritten |
| +// later in the same second T, and every later listing shows stamp T — which |
| +// this cache would read as "unchanged" forever. Refusing to cache anything |
| +// stamped within the last two seconds closes it, at the cost of re-reading a |
| +// just-changed claim once or twice more. Claims change on human timescales, |
| +// so that cost is approximately never paid. |
| +const modSettle = 2 * time.Second |
| + |
| +// List folds the deployment's claims, reading only the ones that changed. |
| +// |
| +// The returned slice is fresh on every call and safe for the caller to keep; |
| +// Claim is a value type with no reference fields, so cached entries cannot |
| +// be mutated through it. |
| +func (c *ClaimCache) List(ctx context.Context, s release.Store) ([]Claim, error) { |
| + lister, ok := s.(release.ModLister) |
| + if c == nil || !ok { |
| + return ListClaims(ctx, s) |
| + } |
| + listed, err := lister.ListMod(ctx, ClaimsPrefix) |
| + if err != nil { |
| + return nil, err |
| + } |
| + now := time.Now() |
| + c.mu.Lock() |
| + defer c.mu.Unlock() |
| + next := make(map[string]cachedClaim, len(listed)) |
| + claims := make([]Claim, 0, len(listed)) |
| + for _, km := range listed { |
| + if hit, ok := c.entries[km.Key]; ok && hit.mod.Equal(km.Mod) { |
| + // Carried forward WITHOUT re-checking modSettle: the bound |
| + // applies when an entry is admitted, so anything already in the |
| + // map was settled when it went in. |
| + next[km.Key] = hit |
| + if !hit.skipped { |
| + claims = append(claims, hit.claim) |
| + } |
| + continue |
| + } |
| + claim, ok, firm := readClaim(ctx, s, km.Key) |
| + if ok { |
| + claims = append(claims, claim) |
| + } |
| + if !firm { |
| + // The read itself failed. Remembering that would hide a good |
| + // claim until something moved its stamp — and nothing edits a |
| + // settled claim, so that is indistinguishable from forever. |
| + continue |
| + } |
| + if now.Sub(km.Mod) < modSettle { |
| + // Too fresh to trust as an unchanged-marker. Used, not kept. |
| + continue |
| + } |
| + next[km.Key] = cachedClaim{mod: km.Mod, claim: claim, skipped: !ok} |
| + } |
| + // Whatever is not in next is not in the bucket: replacing the map whole |
| + // IS the eviction, so a detached claim leaves the cache on the same pass |
| + // it leaves the listing, with no separate invalidation to forget. |
| + c.entries = next |
| + return claims, nil |
| +} |
| + |
| +// Reset drops everything. Nothing in this package needs it; it exists so a |
| +// caller whose store changed underneath it (a test, a reconfiguration) can |
| +// say so explicitly rather than construct a new cache and lose the field. |
| +func (c *ClaimCache) Reset() { |
| + if c == nil { |
| + return |
| + } |
| + c.mu.Lock() |
| + defer c.mu.Unlock() |
| + c.entries = nil |
| +} |
diff --git a/internal/domains/claimcache_test.go b/internal/domains/claimcache_test.go| new file mode 100644 |
| index 000000000..985dffad1 |
| --- /dev/null |
| +++ b/internal/domains/claimcache_test.go |
| @@ -0,0 +1,374 @@ |
| +package domains |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "os" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// countingStore counts the GETs the fold spends. That count IS the property |
| +// under test — the whole point of the cache is that a claim whose bucket |
| +// stamp has not moved costs nothing to fold again. |
| +type countingStore struct { |
| + release.Store |
| + gets int |
| +} |
| + |
| +func (c *countingStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + c.gets++ |
| + return c.Store.Get(ctx, key) |
| +} |
| + |
| +func (c *countingStore) ListMod(ctx context.Context, prefix string) ([]release.KeyMod, error) { |
| + return c.Store.(release.ModLister).ListMod(ctx, prefix) |
| +} |
| + |
| +func seedClaim(t *testing.T, s release.Store, host, app string) { |
| + t.Helper() |
| + if _, _, err := AttachClaim(context.Background(), s, Claim{Host: host, Account: "acct", App: app}); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +// backdate moves a claim's bucket stamp out of the modSettle window, which |
| +// is what a claim written more than a couple of seconds ago looks like. Real |
| +// claims change on human timescales; a test cannot afford to sleep for them. |
| +func backdate(t *testing.T, dir, host string) { |
| + t.Helper() |
| + old := time.Now().Add(-time.Hour) |
| + if err := os.Chtimes(dir+"/"+ClaimKey(host), old, old); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +func TestClaimCacheReReadsNothingWhenNothingChanged(t *testing.T) { |
| + dir := t.TempDir() |
| + s := &countingStore{Store: release.NewDirStore(dir)} |
| + ctx := context.Background() |
| + for _, h := range []string{"a.customer.test", "b.customer.test", "c.customer.test"} { |
| + seedClaim(t, s, h, "kass") |
| + backdate(t, dir, h) |
| + } |
| + s.gets = 0 |
| + |
| + var c ClaimCache |
| + first, err := c.List(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(first) != 3 { |
| + t.Fatalf("first fold = %+v, want 3 claims", first) |
| + } |
| + cold := s.gets |
| + if cold != 3 { |
| + t.Fatalf("cold fold spent %d gets, want one per claim", cold) |
| + } |
| + |
| + s.gets = 0 |
| + second, err := c.List(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(second) != 3 { |
| + t.Fatalf("warm fold = %+v, want the same 3 claims", second) |
| + } |
| + if s.gets != 0 { |
| + t.Errorf("warm fold spent %d gets, want 0 — an unmoved stamp must cost nothing", s.gets) |
| + } |
| +} |
| + |
| +// The cache must be invisible: what it returns has to be what the uncached |
| +// fold returns, claim for claim, or a hostname appears on one read path and |
| +// vanishes on another. |
| +func TestClaimCacheAgreesWithTheUncachedFold(t *testing.T) { |
| + dir := t.TempDir() |
| + s := release.NewDirStore(dir) |
| + ctx := context.Background() |
| + for _, h := range []string{"a.customer.test", "b.customer.test"} { |
| + seedClaim(t, s, h, "kass") |
| + backdate(t, dir, h) |
| + } |
| + var c ClaimCache |
| + if _, err := c.List(ctx, s); err != nil { // prime |
| + t.Fatal(err) |
| + } |
| + cached, err := c.List(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + plain, err := ListClaims(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(cached) != len(plain) { |
| + t.Fatalf("cached fold has %d claims, uncached has %d", len(cached), len(plain)) |
| + } |
| + byHost := map[string]Claim{} |
| + for _, p := range plain { |
| + byHost[p.Host] = p |
| + } |
| + for _, got := range cached { |
| + want, ok := byHost[got.Host] |
| + if !ok { |
| + t.Errorf("cached fold invented %s", got.Host) |
| + continue |
| + } |
| + if got != want { |
| + t.Errorf("%s: cached %+v, uncached %+v", got.Host, got, want) |
| + } |
| + } |
| +} |
| + |
| +// A detached claim leaves the listing, and leaving the listing IS the |
| +// eviction — there is no separate invalidation call to forget to make. |
| +func TestClaimCacheDropsADetachedClaim(t *testing.T) { |
| + dir := t.TempDir() |
| + s := release.NewDirStore(dir) |
| + ctx := context.Background() |
| + seedClaim(t, s, "a.customer.test", "kass") |
| + seedClaim(t, s, "b.customer.test", "kass") |
| + backdate(t, dir, "a.customer.test") |
| + backdate(t, dir, "b.customer.test") |
| + |
| + var c ClaimCache |
| + if got, err := c.List(ctx, s); err != nil || len(got) != 2 { |
| + t.Fatalf("first fold = %+v, %v", got, err) |
| + } |
| + if _, err := DetachClaim(ctx, s, "b.customer.test", "acct", "kass"); err != nil { |
| + t.Fatal(err) |
| + } |
| + got, err := c.List(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(got) != 1 || got[0].Host != "a.customer.test" { |
| + t.Fatalf("after detach = %+v, want only a.customer.test", got) |
| + } |
| +} |
| + |
| +// A changed claim is re-read, and only it. This is the case a cache gets |
| +// wrong: serving a stale body because the listing said the key is still |
| +// there. |
| +func TestClaimCacheReReadsOnlyWhatChanged(t *testing.T) { |
| + dir := t.TempDir() |
| + s := &countingStore{Store: release.NewDirStore(dir)} |
| + ctx := context.Background() |
| + seedClaim(t, s, "a.customer.test", "kass") |
| + seedClaim(t, s, "b.customer.test", "kass") |
| + backdate(t, dir, "a.customer.test") |
| + backdate(t, dir, "b.customer.test") |
| + |
| + var c ClaimCache |
| + if _, err := c.List(ctx, s); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := SetClaimCapabilities(ctx, s, "b.customer.test", "acct", "kass", true, false); err != nil { |
| + t.Fatal(err) |
| + } |
| + backdate(t, dir, "b.customer.test") // settled, but at a DIFFERENT stamp |
| + s.gets = 0 |
| + got, err := c.List(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if s.gets != 1 { |
| + t.Errorf("spent %d gets, want exactly 1 — only the changed claim", s.gets) |
| + } |
| + for _, cl := range got { |
| + if cl.Host == "b.customer.test" && !cl.Wildcard { |
| + t.Error("the changed claim came back stale — the cache served a body the bucket no longer holds") |
| + } |
| + } |
| +} |
| + |
| +// THE lost update the modSettle bound exists for. S3 stamps have one-second |
| +// granularity, so a claim read and then rewritten inside the same second |
| +// carries an unchanged-looking stamp. Caching that read would serve the old |
| +// body forever, and "forever" is not an exaggeration: nothing else would |
| +// ever move the stamp. |
| +func TestClaimCacheRefusesToTrustAStampItJustSaw(t *testing.T) { |
| + dir := t.TempDir() |
| + s := &countingStore{Store: release.NewDirStore(dir)} |
| + ctx := context.Background() |
| + seedClaim(t, s, "a.customer.test", "kass") // written just now: inside modSettle |
| + |
| + var c ClaimCache |
| + if _, err := c.List(ctx, s); err != nil { |
| + t.Fatal(err) |
| + } |
| + s.gets = 0 |
| + if _, err := c.List(ctx, s); err != nil { |
| + t.Fatal(err) |
| + } |
| + if s.gets != 1 { |
| + t.Errorf("spent %d gets, want 1 — a stamp from the last two seconds must not be trusted as unchanged", s.gets) |
| + } |
| +} |
| + |
| +// A store that cannot answer ListMod — the fleet store door's client, every |
| +// test fake — degrades to the plain fold. Not to a weaker cache: a customer |
| +// fleet box is the hardest place to notice a stale claim set. |
| +func TestClaimCacheDegradesOnAStoreWithNoStamps(t *testing.T) { |
| + s := stamplessStore{release.NewDirStore(t.TempDir())} |
| + ctx := context.Background() |
| + seedClaim(t, s, "a.customer.test", "kass") |
| + var c ClaimCache |
| + got, err := c.List(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(got) != 1 { |
| + t.Fatalf("fold = %+v, want the claim", got) |
| + } |
| + // Nothing was cached, so a second fold still sees a change made between |
| + // the two — which is the whole point of degrading rather than guessing. |
| + seedClaim(t, s, "b.customer.test", "kass") |
| + if got, err := c.List(ctx, s); err != nil || len(got) != 2 { |
| + t.Fatalf("second fold = %+v, %v; want both claims", got, err) |
| + } |
| +} |
| + |
| +// stamplessStore hides DirStore's ListMod behind a plain Store, the way the |
| +// fleet door's client and the package's test fakes do. |
| +type stamplessStore struct{ inner release.Store } |
| + |
| +func (s stamplessStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + return s.inner.Get(ctx, key) |
| +} |
| +func (s stamplessStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| + return s.inner.Put(ctx, key, body, ifMatch) |
| +} |
| +func (s stamplessStore) List(ctx context.Context, prefix string) ([]string, error) { |
| + return s.inner.List(ctx, prefix) |
| +} |
| +func (s stamplessStore) ListDirs(ctx context.Context, prefix string) ([]string, error) { |
| + return s.inner.ListDirs(ctx, prefix) |
| +} |
| +func (s stamplessStore) Delete(ctx context.Context, key string) error { |
| + return s.inner.Delete(ctx, key) |
| +} |
| + |
| +// A nil cache is legal and means "no caching", so a caller with none to |
| +// offer passes nil rather than every call site growing a branch. |
| +func TestNilClaimCacheFoldsWithoutCaching(t *testing.T) { |
| + s := release.NewDirStore(t.TempDir()) |
| + ctx := context.Background() |
| + seedClaim(t, s, "a.customer.test", "kass") |
| + var c *ClaimCache |
| + got, err := c.List(ctx, s) |
| + if err != nil || len(got) != 1 { |
| + t.Fatalf("nil cache fold = %+v, %v", got, err) |
| + } |
| + c.Reset() // must not panic |
| +} |
| + |
| +// A transient read failure must not be remembered. Remembering it would |
| +// hide a perfectly good claim until something moved its bucket stamp — and |
| +// nothing edits a settled claim, so for that hostname "until" means never. |
| +// A decode refusal IS remembered, because the same bytes refuse the same way |
| +// every pass and re-reading them forever buys nothing. |
| +func TestClaimCacheNeverRemembersATransientReadFailure(t *testing.T) { |
| + dir := t.TempDir() |
| + inner := release.NewDirStore(dir) |
| + s := &flakyStore{Store: inner} |
| + ctx := context.Background() |
| + seedClaim(t, inner, "a.customer.test", "kass") |
| + backdate(t, dir, "a.customer.test") |
| + |
| + var c ClaimCache |
| + s.failNext = true |
| + got, err := c.List(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(got) != 0 { |
| + t.Fatalf("fold during the blip = %+v, want nothing readable", got) |
| + } |
| + // The blip is over. The claim must come back on the very next fold, |
| + // with no stamp change to prompt it. |
| + got, err = c.List(ctx, s) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(got) != 1 || got[0].Host != "a.customer.test" { |
| + t.Fatalf("after the blip = %+v, want the claim back — a read failure was cached", got) |
| + } |
| +} |
| + |
| +type flakyStore struct { |
| + release.Store |
| + failNext bool |
| +} |
| + |
| +func (f *flakyStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + if f.failNext { |
| + f.failNext = false |
| + return nil, "", errFlaky |
| + } |
| + return f.Store.Get(ctx, key) |
| +} |
| + |
| +func (f *flakyStore) ListMod(ctx context.Context, prefix string) ([]release.KeyMod, error) { |
| + return f.Store.(release.ModLister).ListMod(ctx, prefix) |
| +} |
| + |
| +var errFlaky = errors.New("bucket blip") |
| + |
| +// An object that will never decode is remembered, so a deployment carrying |
| +// one piece of litter under domains/ does not pay a GET for it on every |
| +// fold, on every box, forever. |
| +func TestClaimCacheRemembersAnUndecodableObject(t *testing.T) { |
| + dir := t.TempDir() |
| + s := &countingStore{Store: release.NewDirStore(dir)} |
| + ctx := context.Background() |
| + if _, err := s.Put(ctx, ClaimKey("junk.customer.test"), []byte("not json"), ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + backdate(t, dir, "junk.customer.test") |
| + var c ClaimCache |
| + if got, err := c.List(ctx, s); err != nil || len(got) != 0 { |
| + t.Fatalf("fold = %+v, %v; want nothing", got, err) |
| + } |
| + s.gets = 0 |
| + if _, err := c.List(ctx, s); err != nil { |
| + t.Fatal(err) |
| + } |
| + if s.gets != 0 { |
| + t.Errorf("spent %d gets re-reading an object that will never decode", s.gets) |
| + } |
| +} |
| + |
| +// The entry map is replaced whole rather than merged, so a deployment that |
| +// churns through hostnames does not accumulate an entry per hostname it has |
| +// ever held. Reaching into the field is white-box on purpose: there is no |
| +// behaviour to observe here, only unbounded growth, and unbounded growth |
| +// with no test is how a leak reaches production. |
| +func TestClaimCacheDoesNotGrowWithChurnedHostnames(t *testing.T) { |
| + dir := t.TempDir() |
| + s := release.NewDirStore(dir) |
| + ctx := context.Background() |
| + var c ClaimCache |
| + for i, host := range []string{"one.customer.test", "two.customer.test", "three.customer.test"} { |
| + seedClaim(t, s, host, "kass") |
| + backdate(t, dir, host) |
| + if _, err := c.List(ctx, s); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := DetachClaim(ctx, s, host, "acct", "kass"); err != nil { |
| + t.Fatal(err) |
| + } |
| + if _, err := c.List(ctx, s); err != nil { |
| + t.Fatal(err) |
| + } |
| + c.mu.Lock() |
| + n := len(c.entries) |
| + c.mu.Unlock() |
| + if n != 0 { |
| + t.Fatalf("after churning %d hostname(s) the cache holds %d entries, want 0", i+1, n) |
| + } |
| + } |
| +} |
diff --git a/internal/domains/claims.go b/internal/domains/claims.go| index 7cdb7dacc..ec8c8533e 100644 |
| --- a/internal/domains/claims.go |
| +++ b/internal/domains/claims.go |
| @@ -179,25 +179,46 @@ func ListClaims(ctx context.Context, s release.Store) ([]Claim, error) { |
| } |
| var claims []Claim |
| for _, key := range keys { |
| - raw, _, err := s.Get(ctx, key) |
| - if err != nil { |
| - log.Printf("domains: claim %s: %v", key, err) |
| - continue |
| - } |
| - var c Claim |
| - if err := json.Unmarshal(raw, &c); err != nil { |
| - log.Printf("domains: claim %s: unparseable: %v", key, err) |
| - continue |
| + if c, ok, _ := readClaim(ctx, s, key); ok { |
| + claims = append(claims, c) |
| } |
| - if c.Schema > 1 { |
| - log.Printf("domains: claim %s: schema %d is newer than this binary understands, skipping", key, c.Schema) |
| - continue |
| - } |
| - claims = append(claims, c) |
| } |
| return claims, nil |
| } |
| +// readClaim reads and decodes one claim object. Every refusal is a skip and |
| +// never an error to the caller: one unparseable or too-new object under the |
| +// prefix must not take down the whole fold. |
| +// |
| +// The two failure kinds are told apart, and the distinction is load-bearing |
| +// for ClaimCache. A DECODE refusal (unparseable, or a schema this binary |
| +// does not understand) is deterministic — the same bytes will refuse the |
| +// same way next pass — so it is safe to remember. A READ failure is a |
| +// network blip and is not: remembering it would hide a perfectly good claim |
| +// until something unrelated moved its bucket stamp, which for a claim that |
| +// nobody edits is indistinguishable from forever. So a read failure reports |
| +// firm=false and the cache declines to keep it. |
| +// |
| +// Shared with ClaimCache so the two folds cannot drift into different ideas |
| +// of what counts as a claim — which would show up as a hostname that appears |
| +// on a cached read and vanishes on an uncached one. |
| +func readClaim(ctx context.Context, s release.Store, key string) (c Claim, ok, firm bool) { |
| + raw, _, err := s.Get(ctx, key) |
| + if err != nil { |
| + log.Printf("domains: claim %s: %v", key, err) |
| + return Claim{}, false, false |
| + } |
| + if err := json.Unmarshal(raw, &c); err != nil { |
| + log.Printf("domains: claim %s: unparseable: %v", key, err) |
| + return Claim{}, false, true |
| + } |
| + if c.Schema > 1 { |
| + log.Printf("domains: claim %s: schema %d is newer than this binary understands, skipping", key, c.Schema) |
| + return Claim{}, false, true |
| + } |
| + return c, true, true |
| +} |
| + |
| // hostLabelRe mirrors provision.labelRe (which itself mirrors |
| // domains.accountNameRe — see that var's comment) — pinned HERE rather than |
| // imported from internal/provision, because internal/provision/reconcile.go |
diff --git a/internal/edge/certsweep.go b/internal/edge/certsweep.go| new file mode 100644 |
| index 000000000..c4a710823 |
| --- /dev/null |
| +++ b/internal/edge/certsweep.go |
| @@ -0,0 +1,194 @@ |
| +// Cert sweep (#299 §3f, #243): the issuer removes cert-prefix entries that |
| +// are provably dead — and nothing else. |
| +// |
| +// Two kinds of litter accumulate under control/certs/, for two different |
| +// reasons, and they need two different predicates: |
| +// |
| +// - SPENT CHALLENGE TOKENS (#243). The issuer writes an HTTP-01 response |
| +// or a tls-alpn-01 token cert, validates, and deletes it — but the |
| +// delete is housekeeper-gated, so a stand-down or a crash mid-order |
| +// leaves the token behind permanently. A validation's whole life is |
| +// seconds, so AGE is a complete predicate here. |
| +// - CERTS FOR HOSTS NOBODY SERVES ANY MORE. A detached custom domain's |
| +// cert is never republished (ownsCertName refuses) and never deleted, |
| +// so the object lingers, the mirror serves it to matching SNI until it |
| +// expires, and the expired object then lingers forever. |
| +// |
| +// THE PREDICATE FOR THE SECOND IS EXPIRY, NEVER DETACHMENT, and the reason |
| +// is worth stating at length because the tempting version of this sweep is |
| +// catastrophic. A sweep keyed "no claim exists for this host" would delete |
| +// every non-claim-backed cert in the deployment — every platform host, every |
| +// wildcard PEM, the account keys — and worse, a registry lag or a stale |
| +// edges read would take a LIVE host's certs down fleet-wide through the |
| +// mirror. This repo's standing rule is that inference never destroys owner |
| +// property (internal/domains/reconcile.go §1c), and a certificate is the |
| +// member's property. |
| +// |
| +// So: an entry is deleted only when its own leaf expired long ago. That |
| +// dissolves the "the mirror serves it until expiry" concern by |
| +// construction, and it makes a wrong deletion harmless — the entry was |
| +// already dead, and anything still wanted is re-minted by the warm. |
| +// |
| +// Account keys are never touched, by name and again by shape: they are the |
| +// issuer's identity at the CA, and deleting one costs the deployment its |
| +// ACME registration. |
| +package edge |
| + |
| +import ( |
| + "context" |
| + "log" |
| + "strings" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +const ( |
| + // tokenLitterTTL is how long a spent challenge token may linger before |
| + // the sweep takes it (#243). A validation completes in seconds and |
| + // autocert's own order context is five minutes, so an hour is generous |
| + // by two orders of magnitude — deliberately, because deleting a token |
| + // out from under a live validation burns the order. |
| + tokenLitterTTL = time.Hour |
| + // certSweepParseAge is the pre-filter: an object younger than this is |
| + // never even read, let alone deleted. |
| + // |
| + // It is what keeps the sweep from being the per-object N+1 it would |
| + // otherwise be. autocert renews at ~30 days remaining on a 90-day |
| + // certificate, so a HEALTHY entry is rewritten roughly every 60 days and |
| + // its object stamp never gets this old. An entry that does is one |
| + // nothing is renewing — which is exactly the population worth reading. |
| + // Steady-state cost: one listing, zero reads. |
| + certSweepParseAge = 60 * 24 * time.Hour |
| + // certSweepGrace is how long a leaf must have been EXPIRED before its |
| + // entry is removed. A month, because there is no hurry whatsoever: the |
| + // cost of keeping a dead certificate is bytes, and the cost of removing |
| + // a live one is an outage. |
| + certSweepGrace = 30 * 24 * time.Hour |
| + // certSweepInterval is how often the expiry half runs — the half that |
| + // costs reads. A certificate dead for a month can wait a day. |
| + certSweepInterval = 24 * time.Hour |
| + // certSweepMinInterval is the floor for the whole sweep, token half |
| + // included. It matches the warm's own quiet-path cadence, so an ordinary |
| + // pass sweeps and a capped-drain pass (which comes back every minute |
| + // until a backlog clears) does not. |
| + certSweepMinInterval = warmInterval |
| + // certSweepDeletesPerPass bounds the damage any single pass can do, in |
| + // either direction: a bug here deletes at most this many objects before |
| + // somebody sees the log, and a genuine backlog drains over days rather |
| + // than in one burst against the bucket. |
| + certSweepDeletesPerPass = 50 |
| +) |
| + |
| +// sweep removes provably dead entries under the cert prefix. Issuer-gated by |
| +// its caller; see certWarm.pass. |
| +// |
| +// Never returns an error. Every failure is logged and skipped: a sweep is |
| +// housekeeping, and housekeeping that can fail a pass would take the warm |
| +// down with it. |
| +func (c *certWarm) sweep(ctx context.Context, now time.Time) { |
| + if c.store == nil { |
| + return |
| + } |
| + // Not on a capped-drain pass. When a first-time backlog trips |
| + // warmOrdersPerPass, pass() comes back every minute until it clears, and |
| + // housekeeping does not need doing sixty times an hour — the token TTL |
| + // is an hour and the expiry half is daily. Without this the drain would |
| + // spend a listing of the whole cert prefix per minute for nothing. |
| + if !c.lastSweep.IsZero() && now.Sub(c.lastSweep) < certSweepMinInterval { |
| + return |
| + } |
| + lister, ok := c.store.(release.ModLister) |
| + if !ok { |
| + // No stamps, no age, no sweep. Guessing at an object's age from its |
| + // contents is exactly the inference this file refuses to make. |
| + return |
| + } |
| + ctx, cancel := context.WithTimeout(ctx, certSweepTimeout) |
| + defer cancel() |
| + entries, err := lister.ListMod(ctx, certPrefix) |
| + if err != nil { |
| + log.Printf("edge: cert sweep: could not list %s: %v", certPrefix, err) |
| + return |
| + } |
| + c.lastSweep = now |
| + expiryDue := now.Sub(c.lastExpirySweep) >= certSweepInterval |
| + deletes := 0 |
| + for _, e := range entries { |
| + if deletes >= certSweepDeletesPerPass { |
| + log.Printf("edge: cert sweep: stopping at %d deletions this pass — the rest go next pass", deletes) |
| + break |
| + } |
| + name, ok := strings.CutPrefix(e.Key, certPrefix) |
| + if !ok || name == "" { |
| + continue |
| + } |
| + // Account material, first and unconditionally. Never by age, never |
| + // by expiry, never at all. |
| + if accountKeyName(name) { |
| + continue |
| + } |
| + switch { |
| + case isChallengeToken(name): |
| + if now.Sub(e.Mod) < tokenLitterTTL { |
| + continue |
| + } |
| + if c.deleteEntry(ctx, e.Key, "spent challenge token") { |
| + deletes++ |
| + } |
| + case expiryDue && now.Sub(e.Mod) >= certSweepParseAge: |
| + if c.sweepExpired(ctx, e.Key, name, now) { |
| + deletes++ |
| + } |
| + } |
| + } |
| + if expiryDue { |
| + c.lastExpirySweep = now |
| + } |
| +} |
| + |
| +// certSweepTimeout budgets one sweep. It runs on the warm's own goroutine, |
| +// so a hung bucket must not hold the next warm pass hostage. |
| +const certSweepTimeout = 2 * time.Minute |
| + |
| +// isChallengeToken reports whether name is validation scratch rather than a |
| +// serving artifact — an HTTP-01 response body or a tls-alpn-01 token cert. |
| +func isChallengeToken(name string) bool { |
| + return strings.HasSuffix(name, httpTokenSuffix) || strings.HasSuffix(name, alpnTokenSuffix) |
| +} |
| + |
| +// sweepExpired reads one long-untouched entry and removes it only if its own |
| +// leaf says it is long dead. |
| +// |
| +// Everything that is not a confident yes is a no. An entry that will not |
| +// read, will not parse, or carries no leaf is LEFT ALONE — it may be a shape |
| +// a newer build publishes, and a sweep that deletes what it cannot |
| +// understand is a sweep that deletes tomorrow's format. |
| +func (c *certWarm) sweepExpired(ctx context.Context, key, name string, now time.Time) bool { |
| + raw, _, err := c.store.Get(ctx, key) |
| + if err != nil { |
| + log.Printf("edge: cert sweep: could not read %s: %v", key, err) |
| + return false |
| + } |
| + cert, err := parseCertEntry(raw) |
| + if err != nil || cert.Leaf == nil { |
| + log.Printf("edge: cert sweep: %s does not parse as a certificate, leaving it alone", key) |
| + return false |
| + } |
| + if now.Sub(cert.Leaf.NotAfter) < certSweepGrace { |
| + return false |
| + } |
| + return c.deleteEntry(ctx, key, "certificate expired "+cert.Leaf.NotAfter.UTC().Format("2006-01-02")+" for "+name) |
| +} |
| + |
| +// deleteEntry removes one object and says so. Every deletion under this |
| +// prefix is logged with its reason — there is no such thing as a quiet |
| +// delete of cert material. |
| +func (c *certWarm) deleteEntry(ctx context.Context, key, why string) bool { |
| + if err := c.store.Delete(ctx, key); err != nil { |
| + log.Printf("edge: cert sweep: could not delete %s: %v", key, err) |
| + return false |
| + } |
| + log.Printf("edge: cert sweep: removed %s — %s", key, why) |
| + return true |
| +} |
diff --git a/internal/edge/certsweep_test.go b/internal/edge/certsweep_test.go| new file mode 100644 |
| index 000000000..406cff142 |
| --- /dev/null |
| +++ b/internal/edge/certsweep_test.go |
| @@ -0,0 +1,321 @@ |
| +package edge |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "os" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +func sweepWarm(t *testing.T, s release.Store) *certWarm { |
| + t.Helper() |
| + return newCertWarm(func() []string { return nil }, nil, nil, nil, s) |
| +} |
| + |
| +// seedCertEntry writes one cert-prefix object and backdates its stamp, which |
| +// is how an object that has been sitting in the bucket for months looks. |
| +func seedCertEntry(t *testing.T, dir string, s release.Store, name string, body []byte, age time.Duration) { |
| + t.Helper() |
| + if _, err := s.Put(context.Background(), certKey(name), body, ""); err != nil { |
| + t.Fatal(err) |
| + } |
| + when := time.Now().Add(-age) |
| + if err := os.Chtimes(dir+"/"+certKey(name), when, when); err != nil { |
| + t.Fatal(err) |
| + } |
| +} |
| + |
| +func certGone(t *testing.T, s release.Store, name string) bool { |
| + t.Helper() |
| + _, _, err := s.Get(context.Background(), certKey(name)) |
| + return errors.Is(err, os.ErrNotExist) |
| +} |
| + |
| +func expiredPEM(t *testing.T, host string, expiredAgo time.Duration) []byte { |
| + t.Helper() |
| + cert, key := selfSignedLeaf(t, host, time.Now().Add(-expiredAgo)) |
| + pem, err := encodeWildcardPEM(cert, key) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return pem |
| +} |
| + |
| +// #243: a spent token whose gated delete never ran. Age is a complete |
| +// predicate — a validation's whole life is seconds. |
| +func TestSweepTakesSpentChallengeTokens(t *testing.T) { |
| + dir := t.TempDir() |
| + s := release.NewDirStore(dir) |
| + w := sweepWarm(t, s) |
| + seedCertEntry(t, dir, s, "abc123"+httpTokenSuffix, []byte("key-authorization"), 2*time.Hour) |
| + seedCertEntry(t, dir, s, "www.customer.test"+alpnTokenSuffix, []byte("whatever"), 2*time.Hour) |
| + // A token from a validation that may still be running must survive. |
| + seedCertEntry(t, dir, s, "fresh"+httpTokenSuffix, []byte("key-authorization"), time.Minute) |
| + |
| + w.sweep(context.Background(), time.Now()) |
| + |
| + if !certGone(t, s, "abc123"+httpTokenSuffix) { |
| + t.Error("a spent HTTP-01 token survived the sweep") |
| + } |
| + if !certGone(t, s, "www.customer.test"+alpnTokenSuffix) { |
| + t.Error("a spent tls-alpn-01 token cert survived the sweep") |
| + } |
| + if certGone(t, s, "fresh"+httpTokenSuffix) { |
| + t.Error("a token from a possibly-live validation was deleted — that burns the order") |
| + } |
| +} |
| + |
| +// §3f's whole ruling. The predicate is the leaf's own expiry and nothing |
| +// else: a cert for a host nobody serves any more goes ONLY once it is long |
| +// dead, and a current cert is untouchable however old its object is. |
| +func TestSweepDeletesOnlyLongExpiredCertificates(t *testing.T) { |
| + dir := t.TempDir() |
| + s := release.NewDirStore(dir) |
| + w := sweepWarm(t, s) |
| + old := certSweepParseAge + 24*time.Hour |
| + |
| + seedCertEntry(t, dir, s, "dead.customer.test", expiredPEM(t, "dead.customer.test", certSweepGrace+24*time.Hour), old) |
| + // Expired, but not long enough ago. Patience costs bytes; haste costs |
| + // an outage. |
| + seedCertEntry(t, dir, s, "recent.customer.test", expiredPEM(t, "recent.customer.test", time.Hour), old) |
| + // A CURRENT cert whose object happens to be ancient — a CA with long |
| + // certs, or a renewal that has simply not been needed. |
| + cert, key := selfSignedLeaf(t, "live.customer.test", time.Now().Add(365*24*time.Hour)) |
| + livePEM, err := encodeWildcardPEM(cert, key) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + seedCertEntry(t, dir, s, "live.customer.test", livePEM, old) |
| + |
| + w.sweep(context.Background(), time.Now()) |
| + |
| + if !certGone(t, s, "dead.customer.test") { |
| + t.Error("a certificate that expired a month ago survived") |
| + } |
| + if certGone(t, s, "recent.customer.test") { |
| + t.Error("a certificate that expired an hour ago was deleted — the grace period is not being applied") |
| + } |
| + if certGone(t, s, "live.customer.test") { |
| + t.Fatal("A CURRENT CERTIFICATE WAS DELETED. The predicate is expiry, never object age.") |
| + } |
| +} |
| + |
| +// The account keys are the issuer's identity at the CA. They are written |
| +// once and never rewritten, so they are ALWAYS older than the parse floor — |
| +// which is exactly why they are excluded by name before anything else is |
| +// considered. Deleting one costs the deployment its ACME registration. |
| +func TestSweepNeverTouchesAccountKeys(t *testing.T) { |
| + dir := t.TempDir() |
| + s := release.NewDirStore(dir) |
| + w := sweepWarm(t, s) |
| + ancient := 5 * 365 * 24 * time.Hour |
| + seedCertEntry(t, dir, s, autocertAccountKeyName, []byte("account key material"), ancient) |
| + seedCertEntry(t, dir, s, wildcardAccountKeyName, []byte("account key material"), ancient) |
| + |
| + w.sweep(context.Background(), time.Now()) |
| + |
| + for _, name := range []string{autocertAccountKeyName, wildcardAccountKeyName} { |
| + if certGone(t, s, name) { |
| + t.Fatalf("ACCOUNT KEY %s WAS DELETED — the deployment has lost its ACME registration", name) |
| + } |
| + } |
| +} |
| + |
| +// A sweep that deletes what it cannot understand is a sweep that deletes |
| +// tomorrow's format. Anything that will not read, will not parse, or has no |
| +// leaf is left exactly where it is. |
| +func TestSweepLeavesWhatItCannotUnderstand(t *testing.T) { |
| + dir := t.TempDir() |
| + s := release.NewDirStore(dir) |
| + w := sweepWarm(t, s) |
| + old := certSweepParseAge + 24*time.Hour |
| + seedCertEntry(t, dir, s, "future-shape.customer.test", []byte("a format this build has never seen"), old) |
| + |
| + w.sweep(context.Background(), time.Now()) |
| + |
| + if certGone(t, s, "future-shape.customer.test") { |
| + t.Error("an entry this build could not parse was deleted") |
| + } |
| +} |
| + |
| +// The pre-filter is what keeps this from being an N+1: nothing younger than |
| +// the parse floor is even read. A healthy deployment renews every ~60 days, |
| +// so the steady-state cost is one listing and zero reads. |
| +func TestSweepReadsNothingYoungerThanTheParseFloor(t *testing.T) { |
| + dir := t.TempDir() |
| + s := &bucketGetCounter{Store: release.NewDirStore(dir)} |
| + w := sweepWarm(t, s) |
| + for _, host := range []string{"a.customer.test", "b.customer.test", "c.customer.test"} { |
| + seedCertEntry(t, dir, s, host, expiredPEM(t, host, time.Hour), 10*24*time.Hour) |
| + } |
| + before := s.gets() |
| + w.sweep(context.Background(), time.Now()) |
| + if n := s.gets() - before; n != 0 { |
| + t.Errorf("sweep spent %d get(s) on entries younger than the parse floor, want 0", n) |
| + } |
| +} |
| + |
| +// The expiry half runs daily, not hourly: it is the half that costs reads, |
| +// and a certificate that has been dead for a month can wait a day. The token |
| +// half runs every pass, because it needs no reads at all. |
| +func TestSweepRunsTheExpiryHalfDailyAndTheTokenHalfEveryPass(t *testing.T) { |
| + dir := t.TempDir() |
| + s := release.NewDirStore(dir) |
| + w := sweepWarm(t, s) |
| + old := certSweepParseAge + 24*time.Hour |
| + now := time.Now() |
| + |
| + seedCertEntry(t, dir, s, "dead-one.customer.test", expiredPEM(t, "dead-one.customer.test", certSweepGrace+24*time.Hour), old) |
| + w.sweep(context.Background(), now) |
| + if !certGone(t, s, "dead-one.customer.test") { |
| + t.Fatal("the first sweep did not run the expiry half") |
| + } |
| + |
| + // An hour later, a second dead cert appears and a token goes stale. |
| + seedCertEntry(t, dir, s, "dead-two.customer.test", expiredPEM(t, "dead-two.customer.test", certSweepGrace+24*time.Hour), old) |
| + seedCertEntry(t, dir, s, "tok"+httpTokenSuffix, []byte("key-authorization"), 2*time.Hour) |
| + w.sweep(context.Background(), now.Add(time.Hour)) |
| + if certGone(t, s, "dead-two.customer.test") { |
| + t.Error("the expiry half ran again an hour later — it is meant to be daily") |
| + } |
| + if !certGone(t, s, "tok"+httpTokenSuffix) { |
| + t.Error("the token half did not run on the second pass — it is meant to run every pass") |
| + } |
| + |
| + // A day later it comes back round. |
| + w.sweep(context.Background(), now.Add(certSweepInterval+time.Minute)) |
| + if !certGone(t, s, "dead-two.customer.test") { |
| + t.Error("the expiry half did not run again after a day") |
| + } |
| +} |
| + |
| +// A store with no last-modified stamps gives the sweep no age to judge by, |
| +// and guessing an object's age from its contents is exactly the inference |
| +// this file refuses to make. It does nothing at all rather than something |
| +// approximate. |
| +func TestSweepStandsDownWithoutStamps(t *testing.T) { |
| + dir := t.TempDir() |
| + inner := release.NewDirStore(dir) |
| + w := sweepWarm(t, stamplessEdgeStore{inner}) |
| + seedCertEntry(t, dir, inner, "tok"+httpTokenSuffix, []byte("key-authorization"), 30*24*time.Hour) |
| + |
| + w.sweep(context.Background(), time.Now()) |
| + |
| + if certGone(t, inner, "tok"+httpTokenSuffix) { |
| + t.Error("a store with no stamps still had entries deleted") |
| + } |
| +} |
| + |
| +type stamplessEdgeStore struct{ inner release.Store } |
| + |
| +func (s stamplessEdgeStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| + return s.inner.Get(ctx, key) |
| +} |
| +func (s stamplessEdgeStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| + return s.inner.Put(ctx, key, body, ifMatch) |
| +} |
| +func (s stamplessEdgeStore) List(ctx context.Context, prefix string) ([]string, error) { |
| + return s.inner.List(ctx, prefix) |
| +} |
| +func (s stamplessEdgeStore) ListDirs(ctx context.Context, prefix string) ([]string, error) { |
| + return s.inner.ListDirs(ctx, prefix) |
| +} |
| +func (s stamplessEdgeStore) Delete(ctx context.Context, key string) error { |
| + return s.inner.Delete(ctx, key) |
| +} |
| + |
| +// A bug in the predicate deletes at most this many objects before somebody |
| +// reads the log, and a genuine backlog drains over days rather than in one |
| +// burst against the bucket. |
| +func TestSweepBoundsHowMuchOnePassCanDelete(t *testing.T) { |
| + dir := t.TempDir() |
| + s := release.NewDirStore(dir) |
| + w := sweepWarm(t, s) |
| + total := certSweepDeletesPerPass + 5 |
| + for i := 0; i < total; i++ { |
| + seedCertEntry(t, dir, s, string(rune('a'+i%26))+string(rune('a'+i/26))+".tok"+httpTokenSuffix, |
| + []byte("key-authorization"), 2*time.Hour) |
| + } |
| + w.sweep(context.Background(), time.Now()) |
| + left, err := s.List(context.Background(), certPrefix) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if len(left) != total-certSweepDeletesPerPass { |
| + t.Errorf("%d entries left after one pass, want %d — the per-pass bound is not holding", |
| + len(left), total-certSweepDeletesPerPass) |
| + } |
| +} |
| + |
| +// The sweep's work exists whether or not this deployment has custom |
| +// domains: spent challenge tokens and long-expired certificates are not the |
| +// warm SET's business. A pass over an empty set still sweeps — the gate check |
| +// deliberately comes before the empty-set return in pass(). |
| +func TestPassWithNoWarmSetStillSweeps(t *testing.T) { |
| + dir := t.TempDir() |
| + s := release.NewDirStore(dir) |
| + w := sweepWarm(t, s) |
| + seedCertEntry(t, dir, s, "orphan"+httpTokenSuffix, []byte("key-authorization"), 2*time.Hour) |
| + |
| + if d := w.pass(); d != warmInterval { |
| + t.Fatalf("pass over an empty set returned %s", d) |
| + } |
| + if !certGone(t, s, "orphan"+httpTokenSuffix) { |
| + t.Error("a deployment with no custom domains never sweeps its own token litter") |
| + } |
| +} |
| + |
| +// A non-issuer sweeps nothing. Deleting cert material is the sharpest thing |
| +// this package does, and a box that is not the issuer has no authority to |
| +// do it. |
| +func TestSweepNeverRunsOnANonIssuer(t *testing.T) { |
| + dir := t.TempDir() |
| + s := release.NewDirStore(dir) |
| + w := newCertWarm(func() []string { return nil }, nil, nil, |
| + func() (context.Context, bool) { return nil, false }, s) |
| + seedCertEntry(t, dir, s, "orphan"+httpTokenSuffix, []byte("key-authorization"), 2*time.Hour) |
| + |
| + w.pass() |
| + |
| + if certGone(t, s, "orphan"+httpTokenSuffix) { |
| + t.Error("a stood-down box deleted cert material") |
| + } |
| +} |
| + |
| +// A first-time backlog trips warmOrdersPerPass and pass() comes back every |
| +// minute until it clears. Housekeeping must not ride along sixty times an |
| +// hour: the token TTL is an hour and the expiry half is daily, so a listing |
| +// of the whole cert prefix per minute would be spent on nothing. |
| +func TestSweepDoesNotRunOnEveryCappedDrainPass(t *testing.T) { |
| + dir := t.TempDir() |
| + s := &listCounter{Store: release.NewDirStore(dir)} |
| + w := sweepWarm(t, s) |
| + now := time.Now() |
| + |
| + w.sweep(context.Background(), now) |
| + if s.lists != 1 { |
| + t.Fatalf("first sweep listed %d time(s), want 1", s.lists) |
| + } |
| + w.sweep(context.Background(), now.Add(time.Minute)) |
| + w.sweep(context.Background(), now.Add(2*time.Minute)) |
| + if s.lists != 1 { |
| + t.Errorf("sweep listed %d times across three passes a minute apart, want 1", s.lists) |
| + } |
| + w.sweep(context.Background(), now.Add(certSweepMinInterval+time.Second)) |
| + if s.lists != 2 { |
| + t.Errorf("sweep listed %d times, want 2 — an ordinary pass an hour later must sweep", s.lists) |
| + } |
| +} |
| + |
| +type listCounter struct { |
| + release.Store |
| + lists int |
| +} |
| + |
| +func (l *listCounter) ListMod(ctx context.Context, prefix string) ([]release.KeyMod, error) { |
| + l.lists++ |
| + return l.Store.(release.ModLister).ListMod(ctx, prefix) |
| +} |
diff --git a/internal/edge/certwarm.go b/internal/edge/certwarm.go| index de905e723..3e4700a74 100644 |
| --- a/internal/edge/certwarm.go |
| +++ b/internal/edge/certwarm.go |
| @@ -115,6 +115,13 @@ type certWarm struct { |
| warmed map[string]bool |
| lastStandDownLog time.Time |
| + // lastSweep and lastExpirySweep pace the cert sweep (certsweep.go). |
| + // Deliberately NOT guarded by mu, unlike everything else on this struct: |
| + // they are touched only from sweep(), which is only ever reached from |
| + // pass(), which only ever runs on run()'s single goroutine. A mutex here |
| + // would suggest a concurrency that does not exist. |
| + lastSweep time.Time |
| + lastExpirySweep time.Time |
| } |
| func newCertWarm(hosts func() []string, getCert func(*tls.ClientHelloInfo) (*tls.Certificate, error), pointsHere func(string) bool, issuer IssuerGate, store release.Store) *certWarm { |
| @@ -147,16 +154,21 @@ func (c *certWarm) run() { |
| // pass warms every host in the set once and returns how long to wait. |
| func (c *certWarm) pass() time.Duration { |
| hosts := c.hosts() |
| - if len(hosts) == 0 { |
| - return warmInterval |
| - } |
| // Stand down before ANY work: unlike the wildcard manager there is no |
| // cache to load first, so a non-issuer has nothing useful to do here. |
| + // The gate comes BEFORE the empty-set check, not after, because the |
| + // sweep at the bottom of this pass has work to do in a deployment with |
| + // no custom domains at all — spent challenge tokens and long-expired |
| + // certs are not the warm set's business. |
| ctx, ok := c.issuerGate() |
| if !ok { |
| c.logStandDown(len(hosts)) |
| return warmInterval |
| } |
| + if len(hosts) == 0 { |
| + c.sweep(ctx, time.Now()) |
| + return warmInterval |
| + } |
| // The durable outcome records (#299 §3e), read ONCE per pass. Two things |
| // come out of this read and neither is available in memory: the attempt |
| // count a previous issuer accumulated (so a failover does not restart the |
| @@ -223,6 +235,10 @@ func (c *certWarm) pass() time.Duration { |
| } |
| } |
| c.prune(ctx, records, inSet) |
| + // Housekeeping last: it is the least urgent thing this pass does, and |
| + // running it after the orders means a slow bucket delays litter removal |
| + // rather than certificate issuance (certsweep.go). |
| + c.sweep(ctx, time.Now()) |
| if capped { |
| return warmCappedDelay |
| } |
diff --git a/internal/host/agent.go b/internal/host/agent.go| index 0eab657ac..395fa43ab 100644 |
| --- a/internal/host/agent.go |
| +++ b/internal/host/agent.go |
| @@ -1472,7 +1472,7 @@ func fastConvergeLoop(reg *registry.Registry, svc *activator.Service, sd *schedu |
| // cache would need its own staleness rule, and a STALE route set |
| // is the one thing a security guard must not have. |
| deploymentRoutes := edgesfile.DeploymentRoutes(ctx, store, time.Now()) |
| - if err := domains.ReconcileAliases(ctx, store, reg, time.Now, deploymentRoutes); err != nil { |
| + if err := domains.ReconcileAliases(ctx, store, reg, time.Now, deploymentRoutes, &aliasClaimCache); err != nil { |
| log.Printf("agent: fast converge alias pass: %v", err) |
| } |
| if err := release.Pass(ctx, reg, store, pub, optDir, restart, opts...); err != nil { |
| @@ -1952,7 +1952,7 @@ func instancesLoop(reg *registry.Registry) { |
| } |
| runMovesPass(ctx, store) |
| deploymentRoutes := edgesfile.DeploymentRoutes(ctx, store, time.Now()) |
| - if err := domains.ReconcileAliases(ctx, store, reg, time.Now, deploymentRoutes); err != nil { |
| + if err := domains.ReconcileAliases(ctx, store, reg, time.Now, deploymentRoutes, &aliasClaimCache); err != nil { |
| log.Printf("agent: instances alias pass: %v", err) |
| } |
| if !isFleetBox { |
diff --git a/internal/host/edgesfile.go b/internal/host/edgesfile.go| index 9c4e86bdb..faad8b53f 100644 |
| --- a/internal/host/edgesfile.go |
| +++ b/internal/host/edgesfile.go |
| @@ -14,6 +14,8 @@ import ( |
| "github.com/carlosframework/platform/internal/registry" |
| "github.com/carlosframework/platform/internal/release" |
| + |
| + "github.com/carlosframework/platform/internal/domains" |
| ) |
| // The multi-region ingress spec (2026-08-20, Part 3, closes the design half |
| @@ -404,3 +406,17 @@ func edgesFileLoop(reg *registry.Registry) { |
| } |
| } |
| } |
| + |
| +// aliasClaimCache is the claim fold both alias legs share (the fast-converge |
| +// leg in fullPass and the five-minute instances pass). Process-lifetime and |
| +// package-level for the same reason withheldAliasSaid above is: there is one |
| +// agent per process, and both legs are asking about the same deployment's |
| +// claims, so two caches would double the cold reads and halve the hit rate |
| +// for nothing. |
| +// |
| +// The fast leg is why this exists at all. It runs on every serial bump — a |
| +// seconds cadence — and ListClaims is one List plus a GET per claim in the |
| +// WHOLE deployment. At the hundreds of custom domains titogo is heading for, |
| +// that is a steady drip of bucket reads on every box, sitting ahead of the |
| +// adopt leg inside a two-second convergence promise. See domains.ClaimCache. |
| +var aliasClaimCache domains.ClaimCache |