amadan

carlos / platform Public

Merge pull request #356 from carlosframework/cert-warm-outcomes

warm outcomes: the platform can say a certificate FAILED (#299 §3e)
Paul Campbell pushed by paul@keymail.dev c5e33287a195aeaf91332e9cc9bc627c9c9e2f49 parent c3f021dparent 4861bcd
15 files changed, +1128 −47
  • apimode.go +3 −2
  • deploy.go +13 −0
  • deploy_test.go +33 −0
  • docs/superpowers/specs/2026-08-27-managed-custom-domains-design.md +41 −11
  • internal/certwarm/certwarm.go +189 −0
  • internal/certwarm/certwarm_test.go +104 −0
  • internal/console/apidomains.go +84 −4
  • internal/console/apidomains_test.go +126 −0
  • internal/console/domainstate.go +54 −8
  • internal/console/domainstate_test.go +62 −8
  • internal/edge/certsync.go +8 −1
  • internal/edge/certwarm.go +179 −7
  • internal/edge/certwarm_test.go +220 −3
  • internal/edge/edge.go +1 −0
  • internal/edge/storedoor_test.go +11 −3
diff --git a/apimode.go b/apimode.go
index 118bcc3de..59556c81b 100644
--- a/apimode.go
+++ b/apimode.go
@@ -2526,8 +2526,9 @@ type apiDomainRecord struct {
Catchall bool `json:"catchall"`
DelegationStatus string `json:"delegation_status,omitempty"`
Guidance string `json:"guidance,omitempty"`
- // State is the door's single derived lifecycle word (#299): pending-dns,
- // misdirected, issuing-cert, live or unknown, with Reason (a stable
+ // State is the door's single derived lifecycle word (#299): checking,
+ // pending-dns, misdirected, issuing-cert, failed, live or unknown, with
+ // Reason (a stable
// machine slug) and Detail (a human sentence). Every first-party surface
// reads THIS, not Status.
//
diff --git a/deploy.go b/deploy.go
index 3f0eb4faa..0e40608ce 100644
--- a/deploy.go
+++ b/deploy.go
@@ -1023,6 +1023,18 @@ func dnsGuidanceMessage(domains []apiDomainRecord, host, app string, acct apiAcc
return fmt.Sprintf("DNS for %s is pointing here, but the certificate is not issued yet. "+
"Watch with 'carlos domains list --app %s --account %s'; re-run this deploy shortly — the shipped release is already promoted.",
host, app, acct.Seg), true
+ case d.State == stateFailed:
+ // DNS is right and the certificate is not — and that is ours to
+ // fix, not theirs. The door already derived the honest sentence,
+ // so it is repeated verbatim rather than paraphrased into a
+ // second story about the same hostname. No "Create:" line: their
+ // records are correct, and telling them otherwise would send
+ // them to change the one thing that is working.
+ msg := fmt.Sprintf("DNS for %s is pointing here, but we have not been able to obtain a certificate for it.", host)
+ if d.Detail != "" {
+ msg = fmt.Sprintf("DNS for %s is pointing here. %s", host, d.Detail)
+ }
+ return msg + fmt.Sprintf(" Watch with 'carlos domains list --app %s --account %s' — the shipped release is already promoted.", app, acct.Seg), true
case d.State == stateLive:
// DNS and certificate are both fine, so a connection failure is
// something else entirely and this function has nothing honest
@@ -1476,6 +1488,7 @@ const (
statePendingDNS = "pending-dns"
stateMisdirected = "misdirected"
stateIssuingCert = "issuing-cert"
+ stateFailed = "failed"
stateLive = "live"
stateChecking = "checking"
stateUnknown = "unknown"
diff --git a/deploy_test.go b/deploy_test.go
index cd4665e72..a2f63d08a 100644
--- a/deploy_test.go
+++ b/deploy_test.go
@@ -2161,3 +2161,36 @@ func TestDeployPreflightAsksWhetherTheFlagWasTyped(t *testing.T) {
t.Fatal("no version flag typed, but flagWasSet said yes — the guard would never fire")
}
}
+
+// A deploy that ends with a "failed" domain must not print DNS guidance.
+// Their records are correct — that is precisely what "failed" means — so
+// "Create: ..." would send them to change the one part that is working,
+// while the actual problem (ours) went unnamed.
+func TestDeployDNSGuidanceOnAFailedCertificateNeverAsksForRecords(t *testing.T) {
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ json.NewEncoder(w).Encode(map[string]any{"domains": []map[string]any{{
+ "host": "x.test",
+ "state": "failed",
+ "reason": "cert-failed",
+ "detail": "DNS is correct, but we have not been able to obtain a certificate.",
+ // A console that still stamps guidance alongside it must not
+ // make this print "Create: ..." either.
+ "guidance": "create a CNAME to edge.oncarlos.test",
+ }}})
+ }))
+ defer ts.Close()
+ domains, err := newTestSession(ts.URL).domainList(context.Background(), apiAcct{Seg: "ops"}, "sitely")
+ if err != nil {
+ t.Fatal(err)
+ }
+ msg, ok := dnsGuidanceMessage(domains, "x.test", "sitely", apiAcct{Seg: "ops"})
+ if !ok {
+ t.Fatal("a failed certificate should be reported, not silently dropped")
+ }
+ if strings.Contains(msg, "Create:") || strings.Contains(msg, "CNAME") {
+ t.Errorf("message asks the member to change DNS that is already correct: %q", msg)
+ }
+ if !strings.Contains(msg, "not been able to obtain a certificate") {
+ t.Errorf("message does not name the actual problem: %q", msg)
+ }
+}
diff --git a/docs/superpowers/specs/2026-08-27-managed-custom-domains-design.md b/docs/superpowers/specs/2026-08-27-managed-custom-domains-design.md
index ecd86af12..0870d9ffa 100644
--- a/docs/superpowers/specs/2026-08-27-managed-custom-domains-design.md
+++ b/docs/superpowers/specs/2026-08-27-managed-custom-domains-design.md
@@ -657,19 +657,49 @@ new `aclRead` allowlist entry and an update to
`TestDoorNeverServesCertMaterial` — none of which buy anything the existing
stamps do not already carry.
-### §3e and §3f are follow-ups, not part of this change
+### §3e and §3f were follow-ups; §3e is now built
§3e (per-host `control/cert-warm/<host>.json` CAS records) and §3f (the
-expiry-bounded cert sweep) are not built. The warm works without them: it
-orders, it renews, it backs off per host, and it logs. What is missing is
-durable observability of warm outcomes across a failover, and the pruning of
-certs for long-detached hosts.
-
-Consequence stated plainly: warm attempt counts live in process memory and
-reset when the issuer moves, and `control/certs/` entries for detached hosts
-linger until something else removes them. Neither is a correctness or safety
-problem — §3f's whole ruling was that the sweep must key on EXPIRY rather
-than detachment, so nothing is being deleted wrongly by not running it.
+expiry-bounded cert sweep) shipped after the first change, not with it. The
+warm worked without either: it ordered, it renewed, it backed off per host,
+and it logged. What was missing was durable observability of warm outcomes
+across a failover, and the pruning of certs for long-detached hosts.
+
+**§3e is built (2026-08-29).** `internal/certwarm` owns the wire shape for
+both sides, the issuer writes a record through the same CAS loop `putCert`
+uses (extracted as `putCAS`), and the console folds it into `domainState`.
+Three decisions the spec did not settle, decided here:
+
+- **A record exists only while a host is FAILING.** Success deletes it. That
+ makes absence the healthy case, so the console's fold costs one object per
+ BROKEN hostname rather than one per domain — which is what makes it safe
+ to add to a list door that already pays one Get per claim (#173). The
+ per-host door reads one key instead of listing, preserving the property
+ #328 exists for.
+- **Freshness is judged against `next_attempt_at`, not `written_at`.** A
+ rate-limited host is deliberately not retried for twelve hours, and a bound
+ on write time alone would call that healthy silence stale every time. The
+ accepted limit: an issuer that wedges immediately after writing a
+ rate-limited record is believed for up to fourteen hours. Closing that
+ needs a per-pass heartbeat object — a second key and a second writer-table
+ row for a sharper edge on an already rare case.
+- **`failed` fires at three consecutive failures, or immediately on a rate
+ limit.** Three is about three hours of trying. The rate-limit exception is
+ because the retry is twelve hours away, and a member watching a "usually a
+ minute or two" message for half a day is worse served than one told
+ plainly what is happening.
+
+`failed` never overrides a customer-side condition (`misdirected`,
+`pending-dns`) and never overrides a live certificate: it is only ever the
+alternative to `issuing-cert`. The vocabulary honesty test now produces it
+from an input the platform can actually produce, which is the same bar every
+other state clears.
+
+§3f remains open. Consequence stated plainly: `control/certs/` entries for
+detached hosts linger until something else removes them. That is not a
+correctness or safety problem — §3f's whole ruling was that the sweep must
+key on EXPIRY rather than detachment, so nothing is being deleted wrongly by
+not running it.
### The shared reader is a READER, not the wire-shape owner
diff --git a/internal/certwarm/certwarm.go b/internal/certwarm/certwarm.go
new file mode 100644
index 000000000..de2c55813
--- /dev/null
+++ b/internal/certwarm/certwarm.go
@@ -0,0 +1,189 @@
+// Package certwarm is the wire shape of the cert warm's durable outcome
+// records (#299 §3e): control/cert-warm/<host>.json, written by whichever
+// box holds the issuer lease, read by the console.
+//
+// The record exists so that "the platform tried to get this hostname a
+// certificate and it is not working" can be SAID. Without it the warm's
+// attempt counts live only in the issuer's process memory: they reset on
+// every fleet roll and on every failover, so a domain that has been failing
+// for a week is indistinguishable from one attached ninety seconds ago, and
+// the console has no honest way to show anything but "issuing-cert" forever.
+//
+// One record per host, and ONLY while that host is failing. A successful
+// warm deletes the record, so absence is the healthy case and the prefix
+// holds nothing at all in a deployment where every domain is fine. Two
+// things follow, both load-bearing:
+//
+// - The console's read is cheap by construction. Listing this prefix costs
+// one object per BROKEN host, not one per domain, so folding it into the
+// domains list does not reintroduce the per-claim cost the per-host door
+// (#328) exists to avoid.
+// - "No record" is never evidence of success on its own. It is also what a
+// host that has never been warmed looks like, and what a wedged issuer
+// leaves behind. Readers must treat absence as "nothing to report", and
+// must age-bound what they do find — see Fresh.
+//
+// Unlike internal/edgesfile, this package owns the shape for BOTH sides
+// rather than mirroring a writer that lives elsewhere. The reasoning there
+// was that internal/host's edges-file writer was under heavy development and
+// re-plumbing it onto a shared type would have fought it for no gain. Here
+// writer and reader are being introduced together, in one change, so there
+// is no incumbent to fight and no reason to maintain two decoders of one
+// shape.
+package certwarm
+
+import (
+ "encoding/json"
+ "fmt"
+ "strings"
+ "time"
+)
+
+// Prefix is the flat prefix every record lives under. Inside control/, which
+// the fleet door's aclRead allowlist denies by default — see
+// TestDoorNeverServesCertMaterial, which pins this prefix with the cert
+// material it sits beside.
+const Prefix = "control/cert-warm/"
+
+// V is the wire version this build writes. A reader refuses V<1 (a record
+// with no version is not one of ours) and accepts anything higher, on the
+// same additive-wire rule the edges files use: a newer issuer may add
+// fields, and an older console must degrade to "no verdict" rather than
+// fail the whole fold.
+const V = 1
+
+// Failure classes. Stable machine words — a client may branch on them, so
+// they are part of the wire and are never re-spelled.
+const (
+ // ClassRateLimited: the CA refused because the account's budget is
+ // spent. Materially different from every other failure because it is
+ // nobody's mistake and it will clear on its own, and because the wait is
+ // long enough (12h) that a member deserves to be told rather than shown
+ // a spinner.
+ ClassRateLimited = "rate-limited"
+ // ClassOther: everything else — a failed authorization, a CA outage, a
+ // network error. Deliberately NOT subdivided: the CA's own text is
+ // carried in Error, and inventing a taxonomy over strings we do not
+ // control would go stale the first time the CA rewords a problem
+ // document.
+ ClassOther = "other"
+)
+
+// Key is the bucket key for host's record.
+//
+// The host is used verbatim. Callers hand it the same normalized form the
+// registry and the edges files store — lowercased and punycoded — because a
+// record written under one spelling and read under another is a record that
+// silently does not exist.
+func Key(host string) string { return Prefix + host + ".json" }
+
+// HostFromKey is Key's inverse, for walking a listing. It returns "" for a
+// key outside the prefix or without the .json suffix, so a stray object
+// under the prefix is skipped rather than decoded as a hostname.
+func HostFromKey(key string) string {
+ rest, ok := strings.CutPrefix(key, Prefix)
+ if !ok {
+ return ""
+ }
+ host, ok := strings.CutSuffix(rest, ".json")
+ if !ok || host == "" || strings.Contains(host, "/") {
+ return ""
+ }
+ return host
+}
+
+// Record is one host's warm outcome. Every timestamp is unix seconds.
+type Record struct {
+ V int `json:"v"`
+ Host string `json:"host"`
+ // Attempts is how many consecutive warm attempts have failed since the
+ // last success. It is the field the whole package exists for: it is
+ // carried in the bucket rather than in memory precisely so that a
+ // failover does not reset it to one and restart the clock on a domain
+ // that has been broken for days.
+ Attempts int `json:"attempts"`
+ // FirstFailedAt is when the current run of failures began — preserved
+ // across writes, so "failing since" survives an issuer moving.
+ FirstFailedAt int64 `json:"first_failed_at"`
+ // LastFailedAt is the most recent failure.
+ LastFailedAt int64 `json:"last_failed_at"`
+ // NextAttemptAt is the backoff floor: when the issuer will try again.
+ // Carried because it is the difference between "we have given up" and
+ // "we are waiting out a rate limit", which is the whole of what a member
+ // wants to know, and because Fresh needs it (see there).
+ NextAttemptAt int64 `json:"next_attempt_at"`
+ // Class is one of the Class* constants.
+ Class string `json:"class"`
+ // Error is the CA's or the client's own text, trimmed to ErrorMax. Shown
+ // to operators, never parsed.
+ Error string `json:"error,omitempty"`
+ // WrittenAt is when this record was written. A stamp without its age
+ // cannot be judged — see Fresh.
+ WrittenAt int64 `json:"written_at"`
+}
+
+// ErrorMax bounds the carried error text. A CA problem document can be long,
+// and an unbounded string from a remote party landing in a bucket object
+// that the console renders is exactly the shape that turns one bad response
+// into a large object written every hour.
+const ErrorMax = 512
+
+// Grace is how long past its own next-attempt time a record stays
+// authoritative.
+//
+// The age bound exists because a wedged or stopped issuer keeps its records
+// sitting in the bucket, frozen: without a bound the console would keep
+// reporting "we tried 40 times and stopped" long after nothing is trying at
+// all, which is a worse lie than saying nothing. Judged against
+// NextAttemptAt rather than WrittenAt because the two differ by design — a
+// rate-limited host is deliberately not retried for twelve hours, and a
+// bound on write time alone would call that healthy silence "stale" every
+// time.
+//
+// The limit this accepts, stated plainly: an issuer that wedges immediately
+// after writing a rate-limited record is believed for up to fourteen hours.
+// Closing that would need a per-pass heartbeat object, which is a second
+// key and a second writer-table row to buy a sharper edge on an already
+// rare case.
+const Grace = 2 * time.Hour
+
+// Fresh reports whether this record still describes what the issuer is
+// doing. A stale record is not deleted and not an error — it is simply not
+// used as evidence, and the reader falls back to whatever it would have said
+// with no record at all.
+func (r Record) Fresh(now time.Time) bool {
+ deadline := r.WrittenAt
+ if r.NextAttemptAt > deadline {
+ deadline = r.NextAttemptAt
+ }
+ if deadline == 0 {
+ return false
+ }
+ return now.Before(time.Unix(deadline, 0).Add(Grace))
+}
+
+// Parse decodes one record.
+func Parse(body []byte) (Record, error) {
+ var r Record
+ if err := json.Unmarshal(body, &r); err != nil {
+ return Record{}, fmt.Errorf("cert-warm record: %w", err)
+ }
+ if r.V < V {
+ return Record{}, fmt.Errorf("cert-warm record: unsupported version %d", r.V)
+ }
+ return r, nil
+}
+
+// Encode serializes a record, stamping the version and the written-at time
+// so no caller can forget either.
+func (r Record) Encode(now time.Time) ([]byte, error) {
+ r.V = V
+ r.WrittenAt = now.Unix()
+ if len(r.Error) > ErrorMax {
+ r.Error = r.Error[:ErrorMax]
+ }
+ if r.Class == "" {
+ r.Class = ClassOther
+ }
+ return json.Marshal(r)
+}
diff --git a/internal/certwarm/certwarm_test.go b/internal/certwarm/certwarm_test.go
new file mode 100644
index 000000000..c77fb882e
--- /dev/null
+++ b/internal/certwarm/certwarm_test.go
@@ -0,0 +1,104 @@
+package certwarm
+
+import (
+ "strings"
+ "testing"
+ "time"
+)
+
+func TestKeyAndHostFromKeyRoundTrip(t *testing.T) {
+ for _, host := range []string{"www.customer.com", "xn--bcher-kva.example", "a.b.c.example.org"} {
+ if got := HostFromKey(Key(host)); got != host {
+ t.Errorf("HostFromKey(Key(%q)) = %q", host, got)
+ }
+ }
+}
+
+// A listing under the prefix can contain anything an operator or a future
+// build put there. Everything that is not exactly one host record must come
+// back as "" so the caller skips it rather than treating it as a hostname —
+// a nested key decoded as a host would be pruned as "not in the warm set",
+// which is a delete of somebody else's object.
+func TestHostFromKeyRefusesEverythingElse(t *testing.T) {
+ for _, key := range []string{
+ "control/certs/example.com",
+ "control/cert-warm/",
+ "control/cert-warm/.json",
+ "control/cert-warm/nested/host.json",
+ "control/cert-warm/host.txt",
+ "cert-warm/host.json",
+ "",
+ } {
+ if got := HostFromKey(key); got != "" {
+ t.Errorf("HostFromKey(%q) = %q, want \"\"", key, got)
+ }
+ }
+}
+
+func TestEncodeStampsVersionAndTime(t *testing.T) {
+ now := time.Unix(1_700_000_000, 0)
+ body, err := Record{Host: "a.example"}.Encode(now)
+ if err != nil {
+ t.Fatal(err)
+ }
+ got, err := Parse(body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got.V != V || got.WrittenAt != now.Unix() {
+ t.Errorf("Encode = %+v; want v%d written_at %d", got, V, now.Unix())
+ }
+ // An unset class must never reach the wire: a reader branching on it
+ // would see "" and fall through every case.
+ if got.Class != ClassOther {
+ t.Errorf("Class = %q, want %q", got.Class, ClassOther)
+ }
+}
+
+func TestEncodeTrimsRemoteErrorText(t *testing.T) {
+ body, err := Record{Host: "a.example", Error: strings.Repeat("x", ErrorMax*3)}.Encode(time.Unix(1, 0))
+ if err != nil {
+ t.Fatal(err)
+ }
+ got, err := Parse(body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(got.Error) != ErrorMax {
+ t.Errorf("Error length = %d, want %d — an unbounded CA string reached the bucket", len(got.Error), ErrorMax)
+ }
+}
+
+// V<1 is refused (a record with no version is not one of ours); a HIGHER
+// version parses, because the wire is additive and an older console must
+// degrade to the fields it knows rather than fail the whole fold.
+func TestParseVersionBounds(t *testing.T) {
+ if _, err := Parse([]byte(`{"host":"a.example"}`)); err == nil {
+ t.Error("a record with no version parsed")
+ }
+ got, err := Parse([]byte(`{"v":99,"host":"a.example","attempts":4}`))
+ if err != nil {
+ t.Fatalf("a newer version failed to parse: %v", err)
+ }
+ if got.Attempts != 4 {
+ t.Errorf("Attempts = %d, want 4", got.Attempts)
+ }
+}
+
+// Fresh is judged against the LATER of written-at and next-attempt, which is
+// what lets a deliberate twelve-hour rate-limit wait stay authoritative
+// while a wedged issuer's record eventually stops being believed.
+func TestFreshUsesTheNextAttemptDeadline(t *testing.T) {
+ base := time.Unix(1_700_000_000, 0)
+ rateLimited := Record{WrittenAt: base.Unix(), NextAttemptAt: base.Add(12 * time.Hour).Unix()}
+ if !rateLimited.Fresh(base.Add(6 * time.Hour)) {
+ t.Error("a record mid-way through its own backoff read as stale")
+ }
+ if rateLimited.Fresh(base.Add(12*time.Hour + Grace + time.Second)) {
+ t.Error("a record past its next attempt plus grace still read as fresh — a wedged issuer would report as trying")
+ }
+ // No stamps at all is not evidence of anything.
+ if (Record{}).Fresh(base) {
+ t.Error("an unstamped record read as fresh")
+ }
+}
diff --git a/internal/console/apidomains.go b/internal/console/apidomains.go
index d9595ecc9..5d78f5b13 100644
--- a/internal/console/apidomains.go
+++ b/internal/console/apidomains.go
@@ -46,6 +46,7 @@ import (
"strings"
"time"
+ "github.com/carlosframework/platform/internal/certwarm"
"github.com/carlosframework/platform/internal/domains"
"github.com/carlosframework/platform/internal/edgesfile"
"github.com/carlosframework/platform/internal/provision"
@@ -684,6 +685,12 @@ func (s *Server) handleAPIDomainList(w http.ResponseWriter, r *http.Request) {
}
now := time.Now()
readings := aliasReadings(ctx, s.cfg.Releases, now)
+ // The issuer's warm-outcome records (#299 §3e), also once per call. A
+ // record exists only while a host is FAILING, so this listing is one
+ // object per broken hostname in the deployment and nothing at all in a
+ // deployment where every domain works — it does not reintroduce the
+ // per-claim cost the per-host door exists to avoid.
+ warm := warmRecords(ctx, s.cfg.Releases)
// Read ONCE per call, not per claim: dnsTargets folds the deployment's
// edge set, and a per-claim read would refold it for every hostname.
targets := s.dnsTargets()
@@ -715,7 +722,7 @@ func (s *Server) handleAPIDomainList(w http.ResponseWriter, r *http.Request) {
if delegation {
lookups++
}
- out = append(out, s.domainItem(ctx, c, readings, targets, now, delegation))
+ out = append(out, s.domainItem(ctx, c, readings, warm[c.Host], targets, now, delegation))
}
writeJSON(w, http.StatusOK, map[string]any{
"domains": out,
@@ -738,7 +745,7 @@ func (s *Server) handleAPIDomainList(w http.ResponseWriter, r *http.Request) {
// delegation says whether to spend a live CNAME lookup on this row. The list
// door caps how many it will spend per call; the per-host door always can,
// because it is answering about exactly one host.
-func (s *Server) domainItem(ctx context.Context, c domains.Claim, readings map[string]status.AliasReading, targets DNSTargets, now time.Time, delegation bool) domainListItem {
+func (s *Server) domainItem(ctx context.Context, c domains.Claim, readings map[string]status.AliasReading, warm *certwarm.Record, targets DNSTargets, now time.Time, delegation bool) domainListItem {
item := domainListItem{Host: c.Host, Route: c.Route, Status: "pending",
Wildcard: c.Wildcard, Catchall: c.Catchall}
reading, seen := readings[c.Host]
@@ -748,7 +755,7 @@ func (s *Server) domainItem(ctx context.Context, c domains.Claim, readings map[s
item.CertExpiresAt = reading.CertExpiresAt
item.CheckedAt = reading.DNSCheckedAt
}
- item.State, item.Reason, item.Detail = domainState(reading, seen, now)
+ item.State, item.Reason, item.Detail = domainState(reading, seen, warm, now)
// "unresolved" means the lookup FAILED (our resolver, not the customer's
// record) — pointing them at "create this record" guidance would blame
// them for our outage. Only "" (no answer yet) and "elsewhere" (a real,
@@ -819,7 +826,9 @@ func (s *Server) handleAPIDomainGet(w http.ResponseWriter, r *http.Request) {
}
now := time.Now()
writeJSON(w, http.StatusOK, map[string]any{
- "domain": s.domainItem(ctx, *c, aliasReadings(ctx, s.cfg.Releases, now), s.dnsTargets(), now, c.Wildcard && s.delegationZone != ""),
+ "domain": s.domainItem(ctx, *c, aliasReadings(ctx, s.cfg.Releases, now),
+ warmRecord(ctx, s.cfg.Releases, c.Host), s.dnsTargets(), now,
+ c.Wildcard && s.delegationZone != ""),
"deployment": s.domainCapability(ctx, acct.Sqid, app),
})
}
@@ -876,6 +885,77 @@ func aliasReadings(ctx context.Context, s release.Store, now time.Time) map[stri
return readings
}
+// warmRecords folds the issuer's warm-outcome records (#299 §3e) into a
+// host-keyed map for the LIST door.
+//
+// A record exists only while a host is failing, so the cost is one object
+// per broken hostname in the deployment — usually none. That is the property
+// that makes it safe to fold here at all: the list door already pays one Get
+// per claim (#173), and a second per-claim read would have made a bad cost
+// worse rather than adding a fact.
+//
+// Every failure degrades to "no records". They explain a state; they never
+// gate one, and a domains list that 502s because a telemetry prefix was
+// briefly unreadable would be a worse answer than one that says
+// "issuing-cert".
+func warmRecords(ctx context.Context, s release.Store) map[string]*certwarm.Record {
+ out := map[string]*certwarm.Record{}
+ if s == nil {
+ return out
+ }
+ keys, err := s.List(ctx, certwarm.Prefix)
+ if err != nil {
+ log.Printf("console: api domains: could not list %s: %v — certificate failures will read as in progress", certwarm.Prefix, err)
+ return out
+ }
+ for _, key := range keys {
+ host := certwarm.HostFromKey(key)
+ if host == "" {
+ continue
+ }
+ body, _, err := s.Get(ctx, key)
+ if err != nil {
+ log.Printf("console: api domains: could not read %s: %v", key, err)
+ continue
+ }
+ rec, err := certwarm.Parse(body)
+ if err != nil {
+ // A record a NEWER issuer wrote in a shape this console cannot
+ // read. Skipping it reports "issuing-cert", which is the right
+ // direction to be wrong in.
+ log.Printf("console: api domains: could not parse %s: %v", key, err)
+ continue
+ }
+ out[host] = &rec
+ }
+ return out
+}
+
+// warmRecord reads ONE hostname's record, for the per-host door.
+//
+// Deliberately not warmRecords filtered afterwards: the per-host door's
+// whole reason to exist is that a refresh of one hostname costs one object,
+// and listing a prefix to answer about a single name would give that back.
+// An absent record is the ordinary case and is not an error.
+func warmRecord(ctx context.Context, s release.Store, host string) *certwarm.Record {
+ if s == nil {
+ return nil
+ }
+ body, _, err := s.Get(ctx, certwarm.Key(host))
+ if err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ log.Printf("console: api domains: could not read the warm record for %s: %v", host, err)
+ }
+ return nil
+ }
+ rec, err := certwarm.Parse(body)
+ if err != nil {
+ log.Printf("console: api domains: could not parse the warm record for %s: %v", host, err)
+ return nil
+ }
+ return &rec
+}
+
// hostUnavailableMsg is the ONE refusal string behind every "you cannot have
// this hostname" answer this package gives: the claim already belongs to
// someone else, or a live box in this deployment already serves it as a real
diff --git a/internal/console/apidomains_test.go b/internal/console/apidomains_test.go
index f3cffd366..96c6c0da6 100644
--- a/internal/console/apidomains_test.go
+++ b/internal/console/apidomains_test.go
@@ -25,6 +25,7 @@ import (
"testing"
"time"
+ "github.com/carlosframework/platform/internal/certwarm"
"github.com/carlosframework/platform/internal/domains"
"github.com/carlosframework/platform/internal/edgesfile"
"github.com/carlosframework/platform/internal/provision"
@@ -1694,3 +1695,128 @@ func TestAPIDomainGetRefusesATraversalHost(t *testing.T) {
}
}
}
+
+// The whole of §3e, end to end and through both doors: an issuer's outcome
+// record in the bucket is what turns a hostname that has been trying for
+// hours from "issuing-cert" into "failed". Both doors must reach the same
+// verdict — one reads the record by key, the other folds the prefix, and a
+// consumer refreshing a single host and comparing it against a listed row
+// must never see them disagree.
+func TestAPIDomainFailedStateComesFromTheWarmRecord(t *testing.T) {
+ fx, member := instanceFixture(t, "code-dom-warm")
+ ctx := context.Background()
+ now := time.Now()
+ attachDomain(t, fx, member, "broken.example.com", "go.tito.io")
+ if err := status.WriteBox(ctx, fx.rel, status.Box{
+ GeneratedAt: now, Label: "ada-reporting", Region: "eu-west-1",
+ Metrics: &status.BoxMetrics{Aliases: map[string]status.AliasReading{
+ // DNS is right. No certificate. Without a record this is
+ // "issuing-cert" and stays that way forever.
+ "broken.example.com": {Route: "go.tito.io", DNSStatus: "pointing", DNSCheckedAt: now.Unix()},
+ }},
+ }); err != nil {
+ t.Fatal(err)
+ }
+
+ state := func(t *testing.T) (get, list string) {
+ t.Helper()
+ code, _, out := rawReq(t, http.MethodGet, domainGetURL(fx, "bab", "titogo", "broken.example.com"), member, "")
+ if code != http.StatusOK {
+ t.Fatalf("get = %d %s", code, out)
+ }
+ var one struct {
+ Domain struct{ State string } `json:"domain"`
+ }
+ if err := json.Unmarshal([]byte(out), &one); err != nil {
+ t.Fatal(err)
+ }
+ _, _, listOut := rawReq(t, http.MethodGet, domainsURL(fx, "bab", "titogo"), member, "")
+ var many struct {
+ Domains []struct{ State string } `json:"domains"`
+ }
+ if err := json.Unmarshal([]byte(listOut), &many); err != nil {
+ t.Fatal(err)
+ }
+ if len(many.Domains) != 1 {
+ t.Fatalf("list = %+v", many.Domains)
+ }
+ return one.Domain.State, many.Domains[0].State
+ }
+
+ if get, list := state(t); get != StateIssuingCert || list != StateIssuingCert {
+ t.Fatalf("with no record: get=%q list=%q, want both %q", get, list, StateIssuingCert)
+ }
+
+ writeWarm := func(t *testing.T, rec certwarm.Record) {
+ t.Helper()
+ body, err := rec.Encode(now)
+ if err != nil {
+ t.Fatal(err)
+ }
+ _, etag, err := fx.rel.Get(ctx, certwarm.Key(rec.Host))
+ if err != nil && !errors.Is(err, os.ErrNotExist) {
+ t.Fatal(err)
+ }
+ if _, err := fx.rel.Put(ctx, certwarm.Key(rec.Host), body, etag); err != nil {
+ t.Fatal(err)
+ }
+ }
+
+ // One failed attempt is ordinary and must not change the story.
+ writeWarm(t, certwarm.Record{Host: "broken.example.com", Attempts: 1,
+ Class: certwarm.ClassOther, NextAttemptAt: now.Add(time.Hour).Unix()})
+ if get, list := state(t); get != StateIssuingCert || list != StateIssuingCert {
+ t.Fatalf("after one failure: get=%q list=%q, want both %q", get, list, StateIssuingCert)
+ }
+
+ // At the threshold both doors say so.
+ writeWarm(t, certwarm.Record{Host: "broken.example.com", Attempts: domainFailedAttempts,
+ Class: certwarm.ClassOther, NextAttemptAt: now.Add(time.Hour).Unix()})
+ if get, list := state(t); get != StateFailed || list != StateFailed {
+ t.Fatalf("at the threshold: get=%q list=%q, want both %q", get, list, StateFailed)
+ }
+}
+
+// A record for somebody ELSE's hostname must not reach this app's answer.
+// The cert-warm prefix is deployment-global (there is one issuer, and it
+// writes for every domain it warms), so the join has to be by exact host or
+// one tenant's certificate trouble shows up on another tenant's domain.
+func TestAPIDomainWarmRecordsJoinByExactHost(t *testing.T) {
+ fx, member := instanceFixture(t, "code-dom-warm-iso")
+ ctx := context.Background()
+ now := time.Now()
+ attachDomain(t, fx, member, "mine.example.com", "go.tito.io")
+ if err := status.WriteBox(ctx, fx.rel, status.Box{
+ GeneratedAt: now, Label: "ada-reporting", Region: "eu-west-1",
+ Metrics: &status.BoxMetrics{Aliases: map[string]status.AliasReading{
+ "mine.example.com": {Route: "go.tito.io", DNSStatus: "pointing", DNSCheckedAt: now.Unix()},
+ }},
+ }); err != nil {
+ t.Fatal(err)
+ }
+ body, err := certwarm.Record{Host: "theirs.example.com", Attempts: 99,
+ Class: certwarm.ClassOther, NextAttemptAt: now.Add(time.Hour).Unix()}.Encode(now)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := fx.rel.Put(ctx, certwarm.Key("theirs.example.com"), body, ""); err != nil {
+ t.Fatal(err)
+ }
+ _, _, out := rawReq(t, http.MethodGet, domainsURL(fx, "bab", "titogo"), member, "")
+ var list struct {
+ Domains []struct {
+ Host string `json:"host"`
+ State string `json:"state"`
+ } `json:"domains"`
+ }
+ if err := json.Unmarshal([]byte(out), &list); err != nil {
+ t.Fatal(err)
+ }
+ if len(list.Domains) != 1 || list.Domains[0].Host != "mine.example.com" {
+ t.Fatalf("list = %+v", list.Domains)
+ }
+ if list.Domains[0].State != StateIssuingCert {
+ t.Errorf("state = %q, want %q — another hostname's failure record was joined to this one",
+ list.Domains[0].State, StateIssuingCert)
+ }
+}
diff --git a/internal/console/domainstate.go b/internal/console/domainstate.go
index 861452c89..b0920574d 100644
--- a/internal/console/domainstate.go
+++ b/internal/console/domainstate.go
@@ -5,6 +5,7 @@ import (
"strings"
"time"
+ "github.com/carlosframework/platform/internal/certwarm"
"github.com/carlosframework/platform/internal/status"
)
@@ -42,6 +43,18 @@ const (
// few minutes after an attach, and the other is a fault. Collapsing them
// would make every successful attach open with the word "unknown".
StateChecking = "checking"
+ // StateFailed — the platform tried to obtain a certificate for this
+ // hostname and it is not working.
+ //
+ // Reachable only because the issuer now writes durable warm-outcome
+ // records (#299 §3e, internal/certwarm). Before those existed the
+ // attempt count lived in the issuer's process memory and reset on every
+ // roll and every failover, so a domain broken for a week and one
+ // attached ninety seconds ago looked identical and this word could not
+ // be said honestly. It is the state's whole justification: the thing it
+ // reports is "we have been trying and stopping for hours", which is a
+ // claim about history, and history is what the records carry.
+ StateFailed = "failed"
// StateUnknown — we have no verdict AND that is not routine: our own
// lookup failed, or a box reported something this build cannot read.
// Deliberately distinct from pending-dns: "we could not check" must
@@ -53,16 +66,27 @@ const (
// lifecycle order. Published in the capability block so a client can tell
// which states a deployment might return without guessing from a doc.
//
-// "failed" is deliberately ABSENT. The design reserves it for "the platform
-// tried to obtain a certificate and stopped", which requires the per-host
-// warm-outcome records (§3e) that this build does not write — so nothing can
-// produce it, and advertising a state no deployment will ever return is the
-// same dishonesty that got "verifying" renamed. It is added by the change
-// that makes it reachable, not before.
+// "failed" was absent until the warm-outcome records (§3e) made it
+// reachable, on the rule this list exists to enforce: a deployment
+// advertises only the states it can actually return, because a state nobody
+// will ever see is the same dishonesty that got "verifying" renamed. It is
+// listed now because the issuer writes those records and domainState reads
+// them.
var domainStates = []string{
- StateChecking, StatePendingDNS, StateMisdirected, StateIssuingCert, StateLive, StateUnknown,
+ StateChecking, StatePendingDNS, StateMisdirected, StateIssuingCert, StateFailed, StateLive, StateUnknown,
}
+// domainFailedAttempts is how many consecutive failed warm attempts it takes
+// before a hostname is called "failed" rather than "issuing-cert".
+//
+// Three, because the warm retries hourly: three is about three hours of
+// trying, which is long past any transient the member could have caused and
+// long past a CA blip. One would be wrong in the other direction — a single
+// failed order is ordinary, and telling somebody their domain has failed
+// because the first attempt lost a race would make the word useless the
+// first week it shipped.
+const domainFailedAttempts = 3
+
// domainState derives one hostname's state, a stable machine reason, and a
// human sentence.
//
@@ -70,7 +94,10 @@ var domainStates = []string{
// distinction between "no box mentioned it" and "a box says there is no DNS
// record" is the difference between our problem and the customer's, so it is
// never collapsed.
-func domainState(reading status.AliasReading, seen bool, now time.Time) (state, reason, detail string) {
+// warm is this hostname's durable warm-outcome record, or nil when there is
+// none — and nil is the ordinary case, because a record exists only while a
+// host is failing.
+func domainState(reading status.AliasReading, seen bool, warm *certwarm.Record, now time.Time) (state, reason, detail string) {
if !seen {
return StateChecking, "no-reading",
"No edge has reported on this hostname yet. Checks run every few minutes, so this should settle shortly after attaching."
@@ -91,6 +118,25 @@ func domainState(reading status.AliasReading, seen bool, now time.Time) (state,
return StateLive, "serving",
"This hostname points here and is served over HTTPS with a current certificate."
}
+ // DNS is right and there is no usable certificate. Ordinarily that
+ // is the platform working; the warm record is the only thing that
+ // can distinguish it from the platform failing, and only while it is
+ // FRESH — a frozen record from a wedged issuer must not be read as
+ // "still trying" OR as "gave up".
+ if warm != nil && warm.Fresh(now) {
+ switch {
+ case warm.Class == certwarm.ClassRateLimited:
+ // Reported on the first occurrence, not at the threshold.
+ // The retry is twelve hours away, and a member watching a
+ // "usually a minute or two" message for half a day is worse
+ // served than one told plainly what is happening.
+ return StateFailed, "cert-rate-limited",
+ "Our certificate authority is rate-limiting this deployment, so the certificate for this hostname is waiting. It retries automatically — there is nothing to change on your side."
+ case warm.Attempts >= domainFailedAttempts:
+ return StateFailed, "cert-failed",
+ "DNS is correct, but we have not been able to obtain a certificate for this hostname after several attempts. We keep retrying, and we are looking at it — you do not need to change anything."
+ }
+ }
return StateIssuingCert, "awaiting-certificate",
"DNS is correct. A certificate is being obtained for this hostname — usually a minute or two, and no action is needed."
}
diff --git a/internal/console/domainstate_test.go b/internal/console/domainstate_test.go
index e74e04600..32872401e 100644
--- a/internal/console/domainstate_test.go
+++ b/internal/console/domainstate_test.go
@@ -6,6 +6,7 @@ import (
"testing"
"time"
+ "github.com/carlosframework/platform/internal/certwarm"
"github.com/carlosframework/platform/internal/status"
)
@@ -28,7 +29,7 @@ func TestDomainStateVocabulary(t *testing.T) {
"pointing, live cert": {reading: status.AliasReading{DNSStatus: "pointing", CertExpiresAt: future}, seen: true, wantState: StateLive, wantReason: "serving"},
"a status we don't know": {reading: status.AliasReading{DNSStatus: "quantum"}, seen: true, wantState: StateUnknown, wantReason: "unrecognised-status"},
} {
- state, reason, detail := domainState(tc.reading, tc.seen, now)
+ state, reason, detail := domainState(tc.reading, tc.seen, nil, now)
if state != tc.wantState || reason != tc.wantReason {
t.Errorf("%s: (%q, %q), want (%q, %q)", name, state, reason, tc.wantState, tc.wantReason)
}
@@ -44,8 +45,8 @@ func TestDomainStateVocabulary(t *testing.T) {
// customer for our outage.
func TestDomainStateNeverBlamesTheCustomerForOurOutage(t *testing.T) {
now := time.Unix(1_000_000, 0)
- ourFault, _, ourDetail := domainState(status.AliasReading{DNSStatus: "unresolved"}, true, now)
- theirs, _, _ := domainState(status.AliasReading{DNSStatus: ""}, true, now)
+ ourFault, _, ourDetail := domainState(status.AliasReading{DNSStatus: "unresolved"}, true, nil, now)
+ theirs, _, _ := domainState(status.AliasReading{DNSStatus: ""}, true, nil, now)
if ourFault == theirs {
t.Fatalf("a failed lookup and a missing record both read %q", ourFault)
}
@@ -66,9 +67,18 @@ func TestDomainStateNeverBlamesTheCustomerForOurOutage(t *testing.T) {
func TestAdvertisedStatesAreExactlyTheReachableOnes(t *testing.T) {
now := time.Unix(1_000_000, 0)
produced := map[string]bool{}
+ // The warm inputs are as real as the readings: a fresh record with the
+ // attempt count the threshold names is exactly what the issuer writes,
+ // so "failed" earns its place in the vocabulary the same way every other
+ // state does — by being produced from an input the platform can produce.
+ failing := &certwarm.Record{
+ Attempts: domainFailedAttempts, Class: certwarm.ClassOther,
+ WrittenAt: now.Unix(), NextAttemptAt: now.Unix() + 3600,
+ }
for _, tc := range []struct {
r status.AliasReading
seen bool
+ warm *certwarm.Record
}{
{seen: false},
{r: status.AliasReading{DNSStatus: "unresolved"}, seen: true},
@@ -76,8 +86,9 @@ func TestAdvertisedStatesAreExactlyTheReachableOnes(t *testing.T) {
{r: status.AliasReading{DNSStatus: "elsewhere"}, seen: true},
{r: status.AliasReading{DNSStatus: "pointing"}, seen: true},
{r: status.AliasReading{DNSStatus: "pointing", CertExpiresAt: now.Unix() + 86400}, seen: true},
+ {r: status.AliasReading{DNSStatus: "pointing"}, seen: true, warm: failing},
} {
- s, _, _ := domainState(tc.r, tc.seen, now)
+ s, _, _ := domainState(tc.r, tc.seen, tc.warm, now)
produced[s] = true
}
for _, s := range domainStates {
@@ -90,9 +101,52 @@ func TestAdvertisedStatesAreExactlyTheReachableOnes(t *testing.T) {
t.Errorf("state %q is produced but not advertised", s)
}
}
- // The specific one this build must NOT claim.
- if slices.Contains(domainStates, "failed") {
- t.Error(`"failed" is advertised, but nothing writes the warm-outcome records that would produce it`)
+}
+
+// "failed" is a claim about HISTORY — that the platform has been trying and
+// failing for hours — so the only thing that may produce it is a record of
+// that history. Three ways of having no such record must all still read as
+// "issuing-cert", and each of them was a live bug shape before §3e: no
+// record at all (the ordinary case), a record from an issuer that has since
+// wedged, and a record that has not yet reached the threshold.
+func TestFailedIsOnlyEverSaidOnFreshEnoughEvidence(t *testing.T) {
+ now := time.Unix(1_000_000, 0)
+ pointing := status.AliasReading{DNSStatus: "pointing"}
+ fresh := func(r certwarm.Record) *certwarm.Record {
+ r.WrittenAt, r.NextAttemptAt = now.Unix(), now.Unix()+3600
+ return &r
+ }
+ for _, tc := range []struct {
+ name string
+ warm *certwarm.Record
+ want string
+ }{
+ {"no record at all", nil, StateIssuingCert},
+ {"below the threshold", fresh(certwarm.Record{Attempts: domainFailedAttempts - 1}), StateIssuingCert},
+ {"at the threshold", fresh(certwarm.Record{Attempts: domainFailedAttempts}), StateFailed},
+ {"rate-limited on the first attempt", fresh(certwarm.Record{Attempts: 1, Class: certwarm.ClassRateLimited}), StateFailed},
+ {"stale record from a wedged issuer", &certwarm.Record{
+ Attempts: 40, WrittenAt: now.Add(-30 * 24 * time.Hour).Unix(),
+ NextAttemptAt: now.Add(-30 * 24 * time.Hour).Unix(),
+ }, StateIssuingCert},
+ } {
+ if got, _, _ := domainState(pointing, true, tc.warm, now); got != tc.want {
+ t.Errorf("%s: state = %q, want %q", tc.name, got, tc.want)
+ }
+ }
+ // A live certificate outranks any record. The record is evidence about
+ // an ORDER; the cert is evidence about the hostname, and a hostname that
+ // is being served must never be told it has failed.
+ served := status.AliasReading{DNSStatus: "pointing", CertExpiresAt: now.Unix() + 86400}
+ if got, _, _ := domainState(served, true, fresh(certwarm.Record{Attempts: 99}), now); got != StateLive {
+ t.Errorf("a served hostname with a stale failure record = %q, want %q", got, StateLive)
+ }
+ // A customer-side condition is never overwritten by our cert trouble:
+ // telling somebody their certificate failed when their DNS points at
+ // their old host would send them looking in the wrong place.
+ elsewhere := status.AliasReading{DNSStatus: "elsewhere"}
+ if got, _, _ := domainState(elsewhere, true, fresh(certwarm.Record{Attempts: 99}), now); got != StateMisdirected {
+ t.Errorf("a misdirected hostname with a failure record = %q, want %q", got, StateMisdirected)
}
}
@@ -197,7 +251,7 @@ func TestLegacyStatusStaysDecoupledFromState(t *testing.T) {
now := time.Unix(1_000_000, 0)
// The exact divergence: a box HAS reported (so status is "live"), and
// what it reported is that the hostname points somewhere else.
- state, _, _ := domainState(status.AliasReading{DNSStatus: "elsewhere"}, true, now)
+ state, _, _ := domainState(status.AliasReading{DNSStatus: "elsewhere"}, true, nil, now)
if state != StateMisdirected {
t.Fatalf("state = %q, want %q", state, StateMisdirected)
}
diff --git a/internal/edge/certsync.go b/internal/edge/certsync.go
index 8dbca3f72..46c318f3a 100644
--- a/internal/edge/certsync.go
+++ b/internal/edge/certsync.go
@@ -349,7 +349,14 @@ func publishableCertName(name string) bool {
// function would still race (check-then-put), so the repair loop is the
// guarantee, and it converges because the NotAfter compare is monotone.
func putCert(ctx context.Context, s release.Store, name string, body []byte) error {
- key := certKey(name)
+ return putCAS(ctx, s, certKey(name), body)
+}
+
+// putCAS is putCert's write, by key rather than by cert name — the same
+// read-etag-then-If-Match loop, shared so the cert warm's outcome records
+// (#299 §3e, certwarm.go) get the identical concurrency posture instead of a
+// second copy of it that can drift.
+func putCAS(ctx context.Context, s release.Store, key string, body []byte) error {
for attempt := 0; attempt < 3; attempt++ {
_, etag, err := s.Get(ctx, key)
switch {
diff --git a/internal/edge/certwarm.go b/internal/edge/certwarm.go
index 018cbab10..de905e723 100644
--- a/internal/edge/certwarm.go
+++ b/internal/edge/certwarm.go
@@ -36,6 +36,9 @@ import (
"time"
"golang.org/x/crypto/acme"
+
+ "github.com/carlosframework/platform/internal/certwarm"
+ "github.com/carlosframework/platform/internal/release"
)
const (
@@ -96,6 +99,12 @@ type certWarm struct {
pointsHere func(host string) bool
// issuer gates ORDERING, exactly as it gates the wildcard manager's.
issuer IssuerGate
+ // store is the deployment bucket, for the durable outcome records
+ // (#299 §3e, internal/certwarm). Nil disables recording entirely and
+ // changes nothing else: warming, backoff and logging are unaffected, so
+ // an agent-less `carlos edge` and every test that does not care keep
+ // working exactly as before.
+ store release.Store
mu sync.Mutex
// nextAttempt is the per-host backoff floor.
@@ -108,12 +117,13 @@ type certWarm struct {
lastStandDownLog time.Time
}
-func newCertWarm(hosts func() []string, getCert func(*tls.ClientHelloInfo) (*tls.Certificate, error), pointsHere func(string) bool, issuer IssuerGate) *certWarm {
+func newCertWarm(hosts func() []string, getCert func(*tls.ClientHelloInfo) (*tls.Certificate, error), pointsHere func(string) bool, issuer IssuerGate, store release.Store) *certWarm {
return &certWarm{
hosts: hosts,
getCert: getCert,
pointsHere: pointsHere,
issuer: issuer,
+ store: store,
nextAttempt: map[string]time.Time{},
warmed: map[string]bool{},
}
@@ -142,10 +152,22 @@ func (c *certWarm) pass() time.Duration {
}
// Stand down before ANY work: unlike the wildcard manager there is no
// cache to load first, so a non-issuer has nothing useful to do here.
- if _, ok := c.issuerGate(); !ok {
+ ctx, ok := c.issuerGate()
+ if !ok {
c.logStandDown(len(hosts))
return warmInterval
}
+ // The durable outcome records (#299 §3e), read ONCE per pass. Two things
+ // come out of this read and neither is available in memory: the attempt
+ // count a previous issuer accumulated (so a failover does not restart the
+ // clock on a domain that has been broken for days), and the set of hosts
+ // that currently HAVE a record, which is what makes both the
+ // delete-on-success and the prune below decidable without a Get per host.
+ records := c.loadRecords(ctx)
+ inSet := make(map[string]bool, len(hosts))
+ for _, h := range hosts {
+ inSet[h] = true
+ }
orders, capped := 0, false
for _, h := range hosts {
now := time.Now()
@@ -159,7 +181,8 @@ func (c *certWarm) pass() time.Duration {
// The gate is re-checked BETWEEN orders, not once per pass: a pass
// over a large set can outlive a holding, and every order after the
// holding ends is an order this box has no authority to place.
- if _, ok := c.issuerGate(); !ok {
+ gctx, ok := c.issuerGate()
+ if !ok {
c.logStandDown(len(hosts) - orders)
return warmInterval
}
@@ -179,7 +202,7 @@ func (c *certWarm) pass() time.Duration {
orders++
}
if err := c.warmOne(h); err != nil {
- c.fail(h, err)
+ c.fail(gctx, h, err, records[h])
continue
}
c.mu.Lock()
@@ -190,7 +213,16 @@ func (c *certWarm) pass() time.Duration {
if first {
log.Printf("edge: cert warmed for %s", h)
}
+ // A record exists only while a host is FAILING, so success removes
+ // it. That is what makes "no record" the healthy case and keeps the
+ // console's fold one object per broken host rather than one per
+ // domain (internal/certwarm's package comment).
+ if _, had := records[h]; had {
+ c.clearRecord(gctx, h, "recovered")
+ delete(records, h)
+ }
}
+ c.prune(ctx, records, inSet)
if capped {
return warmCappedDelay
}
@@ -204,19 +236,43 @@ func (c *certWarm) warmOne(host string) error {
return err
}
-// fail arms the per-host backoff, harder when the CA is rate-limiting.
-func (c *certWarm) fail(host string, err error) {
+// fail arms the per-host backoff, harder when the CA is rate-limiting, and
+// records the outcome durably.
+//
+// prior is this host's record as the pass read it (the zero Record when
+// there is none). The attempt count and the "failing since" stamp are
+// carried FORWARD from it rather than recomputed from memory — that is the
+// whole reason the record is in the bucket rather than on the heap.
+func (c *certWarm) fail(ctx context.Context, host string, err error, prior certwarm.Record) {
backoff := warmBackoff
+ class := certwarm.ClassOther
var ae *acme.Error
if errors.As(err, &ae) && ae.ProblemType == "urn:ietf:params:acme:error:rateLimited" {
backoff = warmRateLimitBackoff
+ class = certwarm.ClassRateLimited
log.Printf("edge: cert warm for %s rate-limited by the CA — backing off %s (the whole deployment shares one ACME account)", host, backoff)
} else {
log.Printf("edge: cert warm for %s: %v — retrying in %s", host, err, backoff)
}
+ now := time.Now()
+ next := now.Add(backoff)
c.mu.Lock()
- c.nextAttempt[host] = time.Now().Add(backoff)
+ c.nextAttempt[host] = next
c.mu.Unlock()
+
+ first := prior.FirstFailedAt
+ if first == 0 {
+ first = now.Unix()
+ }
+ c.writeRecord(ctx, certwarm.Record{
+ Host: host,
+ Attempts: prior.Attempts + 1,
+ FirstFailedAt: first,
+ LastFailedAt: now.Unix(),
+ NextAttemptAt: next.Unix(),
+ Class: class,
+ Error: err.Error(),
+ }, now)
}
func (c *certWarm) logStandDown(hosts int) {
@@ -374,3 +430,119 @@ func warmResolver(host string) ([]string, error) {
defer cancel()
return net.DefaultResolver.LookupHost(ctx, host)
}
+
+// ---- the durable outcome records (#299 §3e) ---------------------------------
+//
+// Every one of these degrades to a no-op rather than to a failure. A bucket
+// blip must never stop the warm from warming: the records are how the
+// console explains a failure, not how the fix happens, and a pass that
+// cannot write one has still ordered the certificate.
+
+// warmStoreTimeout budgets the record I/O for one pass. Generous relative to
+// the work (a listing plus a handful of small objects) and tiny relative to
+// the hourly cadence, so a slow bucket delays a pass rather than wedging it.
+const warmStoreTimeout = 30 * time.Second
+
+// loadRecords reads every outcome record in the deployment.
+//
+// The cost is one object per BROKEN host, not one per domain — see the
+// package comment on internal/certwarm — so this stays a listing plus a
+// small handful of Gets in any deployment that is mostly working, and grows
+// only in one that is mostly not.
+//
+// A read failure returns what it has. Partial knowledge is the right answer
+// here in both directions: a host whose record failed to load is treated as
+// having none, so its attempt count restarts at one (an undercount, which
+// only ever delays the console reaching "failed") and its record is left
+// alone by the prune (never a delete on the strength of a read we could not
+// complete).
+func (c *certWarm) loadRecords(ctx context.Context) map[string]certwarm.Record {
+ out := map[string]certwarm.Record{}
+ if c.store == nil {
+ return out
+ }
+ ctx, cancel := context.WithTimeout(ctx, warmStoreTimeout)
+ defer cancel()
+ keys, err := c.store.List(ctx, certwarm.Prefix)
+ if err != nil {
+ log.Printf("edge: cert warm: could not list %s: %v — attempt counts fall back to this process's own", certwarm.Prefix, err)
+ return out
+ }
+ for _, key := range keys {
+ host := certwarm.HostFromKey(key)
+ if host == "" {
+ continue
+ }
+ body, _, err := c.store.Get(ctx, key)
+ if err != nil {
+ log.Printf("edge: cert warm: could not read %s: %v", key, err)
+ continue
+ }
+ rec, err := certwarm.Parse(body)
+ if err != nil {
+ // A record this build cannot read is left in place, not deleted:
+ // it may be a NEWER issuer's, and a rolling fleet has both.
+ log.Printf("edge: cert warm: could not parse %s: %v", key, err)
+ continue
+ }
+ out[host] = rec
+ }
+ return out
+}
+
+// writeRecord publishes one host's outcome, CAS so a transient double-holder
+// cannot half-overwrite the other's record.
+func (c *certWarm) writeRecord(ctx context.Context, rec certwarm.Record, now time.Time) {
+ if c.store == nil {
+ return
+ }
+ body, err := rec.Encode(now)
+ if err != nil {
+ log.Printf("edge: cert warm: could not encode the record for %s: %v", rec.Host, err)
+ return
+ }
+ ctx, cancel := context.WithTimeout(ctx, warmStoreTimeout)
+ defer cancel()
+ if err := putCAS(ctx, c.store, certwarm.Key(rec.Host), body); err != nil {
+ log.Printf("edge: cert warm: could not record the failure for %s: %v", rec.Host, err)
+ }
+}
+
+// clearRecord removes one host's record. why names the occasion in the log —
+// "recovered" or "detached" — because those are opposite news and a single
+// deletion line would hide which one happened.
+func (c *certWarm) clearRecord(ctx context.Context, host, why string) {
+ if c.store == nil {
+ return
+ }
+ ctx, cancel := context.WithTimeout(ctx, warmStoreTimeout)
+ defer cancel()
+ if err := c.store.Delete(ctx, certwarm.Key(host)); err != nil {
+ log.Printf("edge: cert warm: could not clear the %s record for %s: %v", why, host, err)
+ return
+ }
+ log.Printf("edge: cert warm: cleared the outcome record for %s (%s)", host, why)
+}
+
+// prune drops records for hosts that are no longer in the warm set — a
+// detached custom domain, or one whose route went away.
+//
+// This deliberately DOES delete on the strength of an inference ("not in the
+// set"), which §3f forbids for certificates. The distinction is ownership,
+// not caution: a certificate is the member's property and a wrong deletion
+// takes their site down, whereas a warm record is the platform's own note
+// about its own attempts and a wrong deletion costs an attempt count that
+// the next failure rebuilds. The prune is also fenced by pass()'s early
+// return on an empty set, so the one case that would delete EVERYTHING —
+// a warm set that briefly folds to nothing — never reaches here.
+func (c *certWarm) prune(ctx context.Context, records map[string]certwarm.Record, inSet map[string]bool) {
+ if c.store == nil || len(records) == 0 {
+ return
+ }
+ for host := range records {
+ if inSet[host] {
+ continue
+ }
+ c.clearRecord(ctx, host, "detached")
+ }
+}
diff --git a/internal/edge/certwarm_test.go b/internal/edge/certwarm_test.go
index 95aa55b75..26b4d8160 100644
--- a/internal/edge/certwarm_test.go
+++ b/internal/edge/certwarm_test.go
@@ -4,12 +4,16 @@ import (
"context"
"crypto/tls"
"errors"
+ "os"
"strings"
"sync"
"testing"
"time"
"golang.org/x/crypto/acme"
+
+ "github.com/carlosframework/platform/internal/certwarm"
+ "github.com/carlosframework/platform/internal/release"
)
// THE trap this whole file exists to avoid. autocert picks the key type from
@@ -138,6 +142,7 @@ func TestWarmReAsksForAlreadyWarmHostsEveryPass(t *testing.T) {
},
func(string) bool { return true },
nil,
+ nil,
)
for i := 0; i < 3; i++ {
w.pass()
@@ -159,6 +164,7 @@ func TestWarmRenewalReArmIgnoresTheDNSGate(t *testing.T) {
func(*tls.ClientHelloInfo) (*tls.Certificate, error) { calls++; return &tls.Certificate{}, nil },
func(string) bool { return pointing },
nil,
+ nil,
)
w.pass() // first order: gate open, host becomes warmed
pointing = false
@@ -177,6 +183,7 @@ func TestWarmDoesNotOrderForAHostThatDoesNotPointHere(t *testing.T) {
func(*tls.ClientHelloInfo) (*tls.Certificate, error) { calls++; return &tls.Certificate{}, nil },
func(string) bool { return false },
nil,
+ nil,
)
w.pass()
if calls != 0 {
@@ -193,6 +200,7 @@ func TestWarmStandsDownWhenNotTheIssuer(t *testing.T) {
func(*tls.ClientHelloInfo) (*tls.Certificate, error) { calls++; return &tls.Certificate{}, nil },
func(string) bool { return true },
func() (context.Context, bool) { return nil, false },
+ nil,
)
if d := w.pass(); d != warmInterval {
t.Fatalf("delay = %s", d)
@@ -207,9 +215,9 @@ func TestWarmStandsDownWhenNotTheIssuer(t *testing.T) {
// one, and every host shares the one ACME account's budget.
func TestWarmBacksOffHarderOnARateLimit(t *testing.T) {
rateLimited := &acme.Error{ProblemType: "urn:ietf:params:acme:error:rateLimited"}
- w := newCertWarm(nil, nil, nil, nil)
- w.fail("a.test", rateLimited)
- w.fail("b.test", errors.New("connection refused"))
+ w := newCertWarm(nil, nil, nil, nil, nil)
+ w.fail(context.Background(), "a.test", rateLimited, certwarm.Record{})
+ w.fail(context.Background(), "b.test", errors.New("connection refused"), certwarm.Record{})
w.mu.Lock()
defer w.mu.Unlock()
limited, ordinary := time.Until(w.nextAttempt["a.test"]), time.Until(w.nextAttempt["b.test"])
@@ -233,6 +241,7 @@ func TestWarmHonoursPerHostBackoff(t *testing.T) {
},
func(string) bool { return true },
nil,
+ nil,
)
w.pass()
w.pass()
@@ -258,6 +267,7 @@ func TestWarmCapsNewOrdersAndReturnsSoon(t *testing.T) {
func(*tls.ClientHelloInfo) (*tls.Certificate, error) { calls++; return &tls.Certificate{}, nil },
func(string) bool { return true },
nil,
+ nil,
)
d := w.pass()
if calls != warmOrdersPerPass {
@@ -267,3 +277,210 @@ func TestWarmCapsNewOrdersAndReturnsSoon(t *testing.T) {
t.Fatalf("capped pass returned %s, want the short %s so the backlog drains", d, warmCappedDelay)
}
}
+
+// ---- the durable outcome records (#299 §3e) ---------------------------------
+
+func warmRecordIn(t *testing.T, s release.Store, host string) (certwarm.Record, bool) {
+ t.Helper()
+ body, _, err := s.Get(context.Background(), certwarm.Key(host))
+ if errors.Is(err, os.ErrNotExist) {
+ return certwarm.Record{}, false
+ }
+ if err != nil {
+ t.Fatalf("reading the record for %s: %v", host, err)
+ }
+ rec, err := certwarm.Parse(body)
+ if err != nil {
+ t.Fatalf("parsing the record for %s: %v", host, err)
+ }
+ return rec, true
+}
+
+// THE point of §3e. Attempt counts used to live in process memory, so every
+// fleet roll and every failover restarted the clock on a domain that had
+// been broken for days — and the console could never say more than
+// "issuing-cert". The count and the "failing since" stamp must be carried
+// forward from the bucket, not from the heap.
+func TestWarmCarriesAttemptCountsAcrossAFailover(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ newIssuer := func() *certWarm {
+ return newCertWarm(
+ func() []string { return []string{"bad.customer.test"} },
+ func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return nil, errors.New("order failed") },
+ func(string) bool { return true },
+ nil,
+ s,
+ )
+ }
+ first := newIssuer()
+ first.pass()
+ rec, ok := warmRecordIn(t, s, "bad.customer.test")
+ if !ok || rec.Attempts != 1 {
+ t.Fatalf("after one pass: record %+v ok=%v; want attempts 1", rec, ok)
+ }
+ failingSince := rec.FirstFailedAt
+
+ // A brand-new process, with no memory of the first one: exactly what a
+ // roll or a lease handover produces.
+ for i := 2; i <= 4; i++ {
+ w := newIssuer()
+ w.pass()
+ rec, ok = warmRecordIn(t, s, "bad.customer.test")
+ if !ok || rec.Attempts != i {
+ t.Fatalf("failover %d: record %+v ok=%v; want attempts %d — the count reset", i, rec, ok, i)
+ }
+ }
+ if rec.FirstFailedAt != failingSince {
+ t.Errorf("first_failed_at moved from %d to %d — \"failing since\" must survive a failover", failingSince, rec.FirstFailedAt)
+ }
+}
+
+// A rate limit is carried as its own class, and with the twelve-hour retry
+// time, because those are the two facts that make it explicable rather than
+// just another failure.
+func TestWarmRecordsTheRateLimitClassAndItsNextAttempt(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ w := newCertWarm(
+ func() []string { return []string{"limited.customer.test"} },
+ func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+ return nil, &acme.Error{ProblemType: "urn:ietf:params:acme:error:rateLimited"}
+ },
+ func(string) bool { return true },
+ nil,
+ s,
+ )
+ w.pass()
+ rec, ok := warmRecordIn(t, s, "limited.customer.test")
+ if !ok {
+ t.Fatal("no record written for a rate-limited host")
+ }
+ if rec.Class != certwarm.ClassRateLimited {
+ t.Errorf("class = %q, want %q", rec.Class, certwarm.ClassRateLimited)
+ }
+ if until := time.Until(time.Unix(rec.NextAttemptAt, 0)); until < warmRateLimitBackoff-time.Minute {
+ t.Errorf("next attempt in %s, want about %s", until, warmRateLimitBackoff)
+ }
+}
+
+// A record exists only while a host is FAILING. Success removes it, which is
+// what makes absence the healthy case and keeps the console's fold one
+// object per broken host rather than one per domain.
+func TestWarmClearsTheRecordOnRecovery(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ broken := true
+ w := newCertWarm(
+ func() []string { return []string{"flaky.customer.test"} },
+ func(*tls.ClientHelloInfo) (*tls.Certificate, error) {
+ if broken {
+ return nil, errors.New("order failed")
+ }
+ return &tls.Certificate{}, nil
+ },
+ func(string) bool { return true },
+ nil,
+ s,
+ )
+ w.pass()
+ if _, ok := warmRecordIn(t, s, "flaky.customer.test"); !ok {
+ t.Fatal("no record after the failure")
+ }
+ broken = false
+ // The per-host backoff is in memory, so a fresh issuer is the honest way
+ // to reach the next attempt without sleeping an hour.
+ recovered := newCertWarm(w.hosts, w.getCert, w.pointsHere, nil, s)
+ recovered.pass()
+ if rec, ok := warmRecordIn(t, s, "flaky.customer.test"); ok {
+ t.Errorf("record %+v survived recovery — absence must be the healthy case", rec)
+ }
+}
+
+// A detached domain leaves the warm set, and its record goes with it.
+// Deleting on an inference is forbidden for CERTIFICATES (§3f) because those
+// are the member's property; a record of our own attempts is ours to drop.
+func TestWarmPrunesRecordsForHostsThatLeftTheSet(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ hosts := []string{"gone.customer.test", "stays.customer.test"}
+ w := newCertWarm(
+ func() []string { return hosts },
+ func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return nil, errors.New("order failed") },
+ func(string) bool { return true },
+ nil,
+ s,
+ )
+ w.pass()
+ for _, h := range hosts {
+ if _, ok := warmRecordIn(t, s, h); !ok {
+ t.Fatalf("no record for %s after the failure", h)
+ }
+ }
+ hosts = []string{"stays.customer.test"}
+ w.pass()
+ if rec, ok := warmRecordIn(t, s, "gone.customer.test"); ok {
+ t.Errorf("record %+v for a detached host survived the prune", rec)
+ }
+ if _, ok := warmRecordIn(t, s, "stays.customer.test"); !ok {
+ t.Error("the prune took a record for a host still in the set")
+ }
+}
+
+// The one prune case that would delete EVERY record is a warm set that
+// briefly folds to nothing — an edges-file read that came back empty, say.
+// pass() returns before any of this on an empty set, so it can never happen;
+// this pins that, because the guard is one `if` away from being refactored out.
+func TestWarmPruneNeverRunsOnAnEmptySet(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ hosts := []string{"bad.customer.test"}
+ w := newCertWarm(
+ func() []string { return hosts },
+ func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return nil, errors.New("order failed") },
+ func(string) bool { return true },
+ nil,
+ s,
+ )
+ w.pass()
+ hosts = nil
+ w.pass()
+ if _, ok := warmRecordIn(t, s, "bad.customer.test"); !ok {
+ t.Error("an empty warm set pruned every record — a transient fold to nothing must never delete")
+ }
+}
+
+// A non-issuer writes nothing at all. Two boxes both recording their own
+// view of a host would be exactly the whole-file overwrite §3e's per-host
+// CAS exists to avoid, with the added wrinkle that the non-issuer's view is
+// not even real — it places no orders.
+func TestWarmStandDownWritesNoRecords(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ w := newCertWarm(
+ func() []string { return []string{"bad.customer.test"} },
+ func(*tls.ClientHelloInfo) (*tls.Certificate, error) { return nil, errors.New("order failed") },
+ func(string) bool { return true },
+ func() (context.Context, bool) { return nil, false },
+ s,
+ )
+ w.pass()
+ if _, ok := warmRecordIn(t, s, "bad.customer.test"); ok {
+ t.Error("a stood-down box wrote an outcome record")
+ }
+}
+
+// The records are observability, never a precondition. A bucket that cannot
+// be listed or written must not stop the warm from warming: the certificate
+// is the job, the record only explains it.
+func TestWarmKeepsWarmingWhenTheRecordStoreIsBroken(t *testing.T) {
+ s := &failingStore{Store: release.NewDirStore(t.TempDir()), putErr: errors.New("bucket down")}
+ calls := 0
+ w := newCertWarm(
+ func() []string { return []string{"www.customer.test"} },
+ func(*tls.ClientHelloInfo) (*tls.Certificate, error) { calls++; return nil, errors.New("order failed") },
+ func(string) bool { return true },
+ nil,
+ s,
+ )
+ if d := w.pass(); d != warmInterval {
+ t.Fatalf("delay = %s, want the ordinary %s", d, warmInterval)
+ }
+ if calls != 1 {
+ t.Fatalf("GetCertificate calls = %d, want 1 — a broken record store stopped the warm", calls)
+ }
+}
diff --git a/internal/edge/edge.go b/internal/edge/edge.go
index bb5cbf6bc..9c5a1bc42 100644
--- a/internal/edge/edge.go
+++ b/internal/edge/edge.go
@@ -2467,6 +2467,7 @@ func RunWithActivatorReady(args []string, act Activator, cache *Cache, ready fun
perHostCert,
dnsGate(warmResolver, edgeIPs),
e.issuerGate,
+ store,
)
go warm.run()
log.Printf("edge: cert warm on (%s pass) — custom domains are ordered and renewed proactively, not on first handshake", warmInterval)
diff --git a/internal/edge/storedoor_test.go b/internal/edge/storedoor_test.go
index c85458132..cfb98cf8f 100644
--- a/internal/edge/storedoor_test.go
+++ b/internal/edge/storedoor_test.go
@@ -536,7 +536,7 @@ func TestDoorNeverServesCertMaterial(t *testing.T) {
// The real Task 4/5 key shapes (certsync.go's certKey naming: bare host,
// the autocert account key, an HTTP-01 token response, a tls-alpn-01
- // token cert, a wildcard account key) plus the three other control/
+ // token cert, a wildcard account key) plus the four other control/
// entries the multi-region design reserves for the housekeeper alone
// (bucket-writer-audit.md, "Not yet built" table): the mesh key, its
// rotation successor (Task 2's dual-accept — control/edge-mesh-key.next,
@@ -552,8 +552,16 @@ func TestDoorNeverServesCertMaterial(t *testing.T) {
"control/edge-mesh-key",
"control/edge-mesh-key.next",
"control/housekeeper-lease/lock.json",
- }
- // Seed three of the eight with plausible-looking secret bytes: if the ACL
+ // The cert warm's outcome records (#299 §3e, internal/certwarm).
+ // They carry no key material, so the leak they would be is not the
+ // same one as the entries above — it is the EXISTENCE oracle this
+ // test's identical-shape check is really about. A door that answered
+ // differently for a record that exists would let anyone with a fleet
+ // token enumerate which of the deployment's custom domains are
+ // currently failing to get a certificate, tenant boundary and all.
+ "control/cert-warm/example.com.json",
+ }
+ // Seed three of the nine with plausible-looking secret bytes: if the ACL
// ever let a Get through, these are the ones that would leak real
// material instead of a "not found" — and mixing seeded with unseeded
// keys through the identical-shape check below is what proves the door