amadan

carlos / platform Public

Merge pull request #297 from carlosframework/cert-backfill

certsync: the issuer backfills pre-existing certs into control/certs/ (#291)
Paul Campbell pushed by paul@keymail.dev 3e8a67592b71c9281969e616660e1c3397656c4b parent d6cd0e9parent 9484feb
5 files changed, +793 −5
  • docs/superpowers/specs/2026-08-20-multi-region-ingress-design.md +18 −0
  • internal/edge/certbackfill_test.go +506 −0
  • internal/edge/certsync.go +260 −0
  • internal/edge/edge.go +1 −5
  • internal/edge/wildcard.go +8 −0
diff --git a/docs/superpowers/specs/2026-08-20-multi-region-ingress-design.md b/docs/superpowers/specs/2026-08-20-multi-region-ingress-design.md
index 246e84bf6..530fcf771 100644
--- a/docs/superpowers/specs/2026-08-20-multi-region-ingress-design.md
+++ b/docs/superpowers/specs/2026-08-20-multi-region-ingress-design.md
@@ -147,6 +147,24 @@ local certs mirror on their own 30-second list-poll, and
`GetCertificate` serves from that mirror. Renewals run ~30 days early,
so a 30s propagation cadence is never on a deadline.
+*Amendment 2026-08-25 (#291).* Publish-on-Put alone left everything
+minted BEFORE the gate armed out of the bucket (the flagship roll found
+4 objects against ~50 live certs, and region edges served nothing until
+renewal). The issuer therefore also **backfills**: after every
+successful mirror poll, the box holding the gate walks its on-disk
+cache and publishes every live cert-shaped entry the bucket lacks or
+holds an older leaf for; account keys publish only when the bucket has
+none (the deployment's ACME identity is whichever key landed first, and
+a takeover issuer's own key never displaces it); challenge tokens,
+expired and unparseable entries never publish; nor does anything the
+deployment does not own — a per-host entry must pass the registry's
+exact-host check, a wildcard PEM must cover the parent it is named for
+(a long-lived box's disk holds every route ever detached, and DirCache
+temp files a kill left behind). The switch is the strict housekeeper
+gate, re-read per entry — an ungated hand-run edge never publishes.
+Converges in one tick; steady state on the issuer is one directory
+listing plus a stat per entry per poll.
+
**Keys in the bucket — the trust statement.** Any box with deployment-
bucket read already serves this deployment's traffic and reads its
releases; holding TLS keys does not extend what a compromised box can
diff --git a/internal/edge/certbackfill_test.go b/internal/edge/certbackfill_test.go
new file mode 100644
index 000000000..30f7e6e6e
--- /dev/null
+++ b/internal/edge/certbackfill_test.go
@@ -0,0 +1,506 @@
+package edge
+
+import (
+ "context"
+ "errors"
+ "io"
+ "log"
+ "os"
+ "path/filepath"
+ "slices"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "golang.org/x/crypto/acme/autocert"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// issuerMirror is a mirror over local+bucket whose box holds the issuer gate
+// when held is true. Refresh has run, so the snapshot reflects the bucket.
+func issuerMirror(t *testing.T, s release.Store, local autocert.Cache, held bool) *certMirror {
+ t.Helper()
+ m := newCertMirror(s, local)
+ m.issuing = func() bool { return held }
+ // The registry: every *.example.com name except "gone".
+ m.allowed = func(host string) bool {
+ return strings.HasSuffix(host, ".example.com") && !strings.HasPrefix(host, "gone.")
+ }
+ // The wildcard manager's parent set: acct only.
+ m.wildcardParents = func() []string { return []string{"acct.example.com"} }
+ if err := m.Refresh(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ return m
+}
+
+func putLocal(t *testing.T, c autocert.Cache, name string, body []byte) {
+ t.Helper()
+ if err := c.Put(context.Background(), name, body); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func bucketHas(t *testing.T, s release.Store, name string) ([]byte, bool) {
+ t.Helper()
+ b, err := certStoreGet(t, s, certKey(name))
+ if err != nil {
+ return nil, false
+ }
+ return b, true
+}
+
+// The flagship roll's shape: a cache full of certs minted before publishing
+// existed, a bucket holding almost nothing. One pass and the bucket has
+// every live cert — and nothing that should not be there.
+func TestBackfillPublishesPreExistingCertsOnce(t *testing.T) {
+ log.SetOutput(io.Discard)
+ t.Cleanup(func() { log.SetOutput(os.Stderr) })
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ later := time.Now().Add(60 * 24 * time.Hour)
+
+ live := hostCertPEM(t, "a.example.com", later)
+ putLocal(t, local, "a.example.com", live)
+ putLocal(t, local, "a.example.com+rsa", hostCertPEM(t, "a.example.com", later))
+ putLocal(t, local, "wildcard-acct.example.com.pem", hostCertPEM(t, "*.acct.example.com", later))
+ putLocal(t, local, "expired.example.com", hostCertPEM(t, "expired.example.com", time.Now().Add(-time.Hour)))
+ putLocal(t, local, "garbage.example.com", []byte("not a cert"))
+ putLocal(t, local, "a.example.com+http-01", []byte("token-body"))
+ putLocal(t, local, "a.example.com+token", hostCertPEM(t, "a.example.com", later))
+ // Disk residue that is NOT the deployment's to replicate: a route
+ // detached long ago, a DirCache temp file a kill left behind, a
+ // wildcard PEM named for a parent its leaf does not cover.
+ putLocal(t, local, "gone.example.com", hostCertPEM(t, "gone.example.com", later))
+ putLocal(t, local, "a.example.com1762384410", live)
+ putLocal(t, local, "wildcard-other.example.com.pem", hostCertPEM(t, "*.acct.example.com", later))
+ putLocal(t, local, "wildcard-left.example.com.pem", hostCertPEM(t, "*.left.example.com", later)) // parent no longer current
+ putLocal(t, local, "acme_account+key", []byte("LOCAL-ACCOUNT"))
+ putLocal(t, local, wildcardAccountKeyName, []byte("LOCAL-WILDCARD-ACCOUNT"))
+
+ m := issuerMirror(t, s, local, true)
+ m.backfill(context.Background())
+
+ for _, name := range []string{"a.example.com", "a.example.com+rsa", "wildcard-acct.example.com.pem"} {
+ if _, ok := bucketHas(t, s, name); !ok {
+ t.Errorf("%s not backfilled", name)
+ }
+ }
+ if got, _ := bucketHas(t, s, "a.example.com"); string(got) != string(live) {
+ t.Errorf("a.example.com body differs from the local cache")
+ }
+ for _, name := range []string{"expired.example.com", "garbage.example.com", "a.example.com+http-01", "a.example.com+token",
+ "gone.example.com", "a.example.com1762384410", "wildcard-other.example.com.pem", "wildcard-left.example.com.pem"} {
+ if _, ok := bucketHas(t, s, name); ok {
+ t.Errorf("%s must not be published", name)
+ }
+ }
+ // Account keys: the bucket had none, so this box's become the
+ // deployment's.
+ for name, want := range map[string]string{"acme_account+key": "LOCAL-ACCOUNT", wildcardAccountKeyName: "LOCAL-WILDCARD-ACCOUNT"} {
+ if got, ok := bucketHas(t, s, name); !ok || string(got) != want {
+ t.Errorf("%s: got %q, %v; want seeded", name, got, ok)
+ }
+ }
+
+ // Steady state: a second pass after a poll writes nothing. A fresh
+ // store wrapper counts.
+ if err := m.Refresh(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ c := &putCountingStore{Store: s}
+ m.s = c
+ m.backfill(context.Background())
+ if c.puts != 0 {
+ t.Fatalf("second pass published %d entries; want 0", c.puts)
+ }
+}
+
+type putCountingStore struct {
+ release.Store
+ puts int
+}
+
+func (c *putCountingStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) {
+ c.puts++
+ return c.Store.Put(ctx, key, body, ifMatch)
+}
+
+// A non-issuer's disk is never the truth: the writer map gives the prefix
+// to the housekeeper alone.
+func TestBackfillIsIssuerOnly(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour)))
+
+ m := issuerMirror(t, s, local, false)
+ m.backfill(context.Background())
+ if _, ok := bucketHas(t, s, "a.example.com"); ok {
+ t.Fatal("a non-issuer published")
+ }
+
+ m2 := newCertMirror(s, local) // issuing nil: never the issuer
+ m2.backfill(context.Background())
+ if _, ok := bucketHas(t, s, "a.example.com"); ok {
+ t.Fatal("a gate-less mirror published")
+ }
+
+ // The Edge wiring is STRICT: a box with a store but no agent gate is
+ // ungated for serving (issuerGate says held) yet is nobody's issuer.
+ e := &Edge{staticStore: s}
+ m3 := e.newCertMirror(local)
+ if m3.issuing() {
+ t.Fatal("an ungated edge counts as the issuer for backfill")
+ }
+ e.issuer = func() (context.Context, bool) { return context.Background(), true }
+ if !m3.issuing() {
+ t.Fatal("a holding edge is not the issuer")
+ }
+}
+
+// The gate is re-read per entry: a holding that ends mid-pass stops the
+// pass where it stands.
+func TestBackfillStopsWhenTheGateIsLostMidPass(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ later := time.Now().Add(time.Hour)
+ for _, h := range []string{"a.example.com", "b.example.com", "c.example.com"} {
+ putLocal(t, local, h, hostCertPEM(t, h, later))
+ }
+ var buf strings.Builder
+ log.SetOutput(&buf)
+ t.Cleanup(func() { log.SetOutput(os.Stderr) })
+ m := issuerMirror(t, s, local, true)
+ calls := 0
+ m.issuing = func() bool { calls++; return calls <= 2 } // the top check, then one entry
+ m.backfill(context.Background())
+ keys, _ := s.List(context.Background(), certPrefix)
+ if len(keys) != 1 {
+ t.Fatalf("published %v after the gate was lost; want exactly one", keys)
+ }
+ if !strings.Contains(buf.String(), "backfilled 1 cache entry") {
+ t.Fatalf("a pass cut short by the gate did not report what it published:\n%s", buf.String())
+ }
+}
+
+// The other cert shapes ride the same freshness path as plain hosts.
+func TestBackfillWildcardAndRSAThroughTheFreshnessPath(t *testing.T) {
+ log.SetOutput(io.Discard)
+ t.Cleanup(func() { log.SetOutput(os.Stderr) })
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ soon, later := time.Now().Add(24*time.Hour), time.Now().Add(48*time.Hour)
+
+ putBucket(t, s, "wildcard-acct.example.com.pem", hostCertPEM(t, "*.acct.example.com", soon))
+ newWild := hostCertPEM(t, "*.acct.example.com", later)
+ putLocal(t, local, "wildcard-acct.example.com.pem", newWild)
+
+ putBucket(t, s, "a.example.com+rsa", hostCertPEM(t, "a.example.com", soon))
+ newRSA := hostCertPEM(t, "a.example.com", later)
+ putLocal(t, local, "a.example.com+rsa", newRSA)
+
+ // A bucket object that does not parse is replaced by a live local one.
+ putBucket(t, s, "b.example.com", []byte("corrupt"))
+ goodB := hostCertPEM(t, "b.example.com", later)
+ putLocal(t, local, "b.example.com", goodB)
+
+ m := issuerMirror(t, s, local, true)
+ m.backfill(context.Background())
+ for name, want := range map[string][]byte{"wildcard-acct.example.com.pem": newWild, "a.example.com+rsa": newRSA, "b.example.com": goodB} {
+ if got, _ := bucketHas(t, s, name); string(got) != string(want) {
+ t.Errorf("%s: bucket copy not brought forward", name)
+ }
+ }
+}
+
+type failingPutStore struct {
+ release.Store
+ mu sync.Mutex
+ puts int
+}
+
+func (f *failingPutStore) Put(context.Context, string, []byte, string) (string, error) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.puts++
+ return "", errors.New("AccessDenied")
+}
+
+// A write-denied bucket retries every pass and says so on the file's
+// first-and-every-tenth cadence, never every 30 seconds.
+func TestBackfillFailureIsRetriedAndThrottled(t *testing.T) {
+ var buf strings.Builder
+ log.SetOutput(&buf)
+ t.Cleanup(func() { log.SetOutput(os.Stderr) })
+ inner := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour)))
+ m := issuerMirror(t, inner, local, true)
+ f := &failingPutStore{Store: inner}
+ m.s = f
+ for i := 0; i < 12; i++ {
+ m.backfill(context.Background())
+ }
+ if f.puts != 12 {
+ t.Fatalf("puts = %d; want one retry per pass", f.puts)
+ }
+ if n := strings.Count(buf.String(), "not published"); n != 2 {
+ t.Fatalf("logged %d times over 12 failing passes; want 2 (#1, #10):\n%s", n, buf.String())
+ }
+}
+
+// Steady state does not re-read the disk: an entry whose stamp has not
+// moved since it matched the bucket is skipped on the listing alone.
+func TestBackfillSteadyStateDoesNotReread(t *testing.T) {
+ log.SetOutput(io.Discard)
+ t.Cleanup(func() { log.SetOutput(os.Stderr) })
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour)))
+ m := issuerMirror(t, s, local, true)
+ m.backfill(context.Background())
+ if err := m.Refresh(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ // Residue the pass settles as not-owned must not be re-read either.
+ putLocal(t, local, "gone.example.com", hostCertPEM(t, "gone.example.com", time.Now().Add(time.Hour)))
+ m.backfill(context.Background())
+ if os.Getuid() == 0 {
+ t.Skip("root ignores mode bits")
+ }
+ // Listing and stat still work; any read now fails loudly.
+ for _, name := range []string{"a.example.com", "gone.example.com"} {
+ if err := os.Chmod(filepath.Join(string(local), name), 0o000); err != nil {
+ t.Fatal(err)
+ }
+ }
+ var buf strings.Builder
+ log.SetOutput(&buf)
+ c := &putCountingStore{Store: s}
+ m.s = c
+ m.backfill(context.Background())
+ if c.puts != 0 {
+ t.Fatalf("steady state published %d", c.puts)
+ }
+ if strings.Contains(buf.String(), "not published") {
+ t.Fatalf("steady state re-read the disk:\n%s", buf.String())
+ }
+ // And a moved stamp IS re-read: touch one and the read fails visibly.
+ now := time.Now().Add(time.Second)
+ if err := os.Chtimes(filepath.Join(string(local), "a.example.com"), now, now); err != nil {
+ t.Fatal(err)
+ }
+ m.backfill(context.Background())
+ if !strings.Contains(buf.String(), "not published") {
+ t.Fatal("a changed stamp was not re-read")
+ }
+}
+
+// autocert writes the cache while the backfill walks it. Whatever the
+// interleaving, nothing but complete, owned entries reach the bucket.
+func TestBackfillRacesAConcurrentIssuance(t *testing.T) {
+ log.SetOutput(io.Discard)
+ t.Cleanup(func() { log.SetOutput(os.Stderr) })
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ later := time.Now().Add(time.Hour)
+ pems := map[string][]byte{}
+ for i := 0; i < 8; i++ {
+ h := string(rune('a'+i)) + ".example.com"
+ pems[h] = hostCertPEM(t, h, later)
+ }
+ m := issuerMirror(t, s, local, true)
+ var wg sync.WaitGroup
+ wg.Add(2)
+ go func() {
+ defer wg.Done()
+ for h, pem := range pems {
+ putLocal(t, local, h, pem)
+ }
+ }()
+ go func() {
+ defer wg.Done()
+ for i := 0; i < 20; i++ {
+ m.backfill(context.Background())
+ }
+ }()
+ wg.Wait()
+ m.backfill(context.Background())
+ keys, _ := s.List(context.Background(), certPrefix)
+ for _, k := range keys {
+ name := strings.TrimPrefix(k, certPrefix)
+ want, ok := pems[name]
+ if !ok {
+ t.Errorf("stray bucket key %s", k)
+ continue
+ }
+ if got, _ := bucketHas(t, s, name); string(got) != string(want) {
+ t.Errorf("%s: bucket body is not the cache's", name)
+ }
+ }
+ if len(keys) != len(pems) {
+ t.Fatalf("bucket has %d of %d", len(keys), len(pems))
+ }
+}
+
+// The renewed-before-the-gate case: the bucket's copy is older than the
+// disk's, so the disk wins; and the reverse — the bucket is fresher — leaves
+// the bucket alone (the mirror will bring the disk up instead).
+func TestBackfillReplacesOnlyAnOlderBucketCert(t *testing.T) {
+ log.SetOutput(io.Discard)
+ t.Cleanup(func() { log.SetOutput(os.Stderr) })
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ soon := time.Now().Add(24 * time.Hour)
+ later := time.Now().Add(48 * time.Hour)
+
+ oldPEM := hostCertPEM(t, "old.example.com", soon)
+ newPEM := hostCertPEM(t, "old.example.com", later)
+ putBucket(t, s, "old.example.com", oldPEM)
+ putLocal(t, local, "old.example.com", newPEM)
+
+ freshBucket := hostCertPEM(t, "fresh.example.com", later)
+ putBucket(t, s, "fresh.example.com", freshBucket)
+ putLocal(t, local, "fresh.example.com", hostCertPEM(t, "fresh.example.com", soon))
+
+ m := issuerMirror(t, s, local, true)
+ m.backfill(context.Background())
+
+ if got, _ := bucketHas(t, s, "old.example.com"); string(got) != string(newPEM) {
+ t.Error("an older bucket cert was not replaced by the disk's newer one")
+ }
+ if got, _ := bucketHas(t, s, "fresh.example.com"); string(got) != string(freshBucket) {
+ t.Error("a fresher bucket cert was clobbered by an older local one")
+ }
+}
+
+// The deployment's ACME identity is whichever key the bucket already
+// holds — a takeover issuer's own key must never displace it.
+func TestBackfillNeverReplacesABucketAccountKey(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ putBucket(t, s, "acme_account+key", []byte("DEPLOYMENT"))
+ putBucket(t, s, wildcardAccountKeyName, []byte("DEPLOYMENT-WILDCARD"))
+ // Refresh seeds the bucket keys locally only when absent; make the local
+ // ones differ first.
+ putLocal(t, local, "acme_account+key", []byte("MINE"))
+ putLocal(t, local, wildcardAccountKeyName, []byte("MINE-WILDCARD"))
+
+ m := issuerMirror(t, s, local, true)
+ m.backfill(context.Background())
+ if got, _ := bucketHas(t, s, "acme_account+key"); string(got) != "DEPLOYMENT" {
+ t.Fatalf("account key replaced: %q", got)
+ }
+ if got, _ := bucketHas(t, s, wildcardAccountKeyName); string(got) != "DEPLOYMENT-WILDCARD" {
+ t.Fatalf("wildcard account key replaced: %q", got)
+ }
+}
+
+// Only an on-disk cache can be enumerated; anything else is a quiet no-op,
+// never a panic in the poll loop.
+func TestBackfillNeedsADirCache(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ m := issuerMirror(t, s, nil, true)
+ m.backfill(context.Background())
+ m2 := issuerMirror(t, s, newCountingCache(t), true)
+ m2.backfill(context.Background())
+ m3 := issuerMirror(t, s, autocert.DirCache(t.TempDir()+"/missing"), true)
+ m3.backfill(context.Background())
+ if keys, _ := s.List(context.Background(), certPrefix); len(keys) != 0 {
+ t.Fatalf("published %v", keys)
+ }
+}
+
+// The poll loop runs the backfill after each successful refresh: an issuer
+// that boots with a full cache and an empty bucket fills it without waiting
+// for a renewal.
+func TestCertMirrorLoopBackfillsOnTheIssuer(t *testing.T) {
+ prev := certMirrorInterval
+ certMirrorInterval = 5 * time.Millisecond
+ t.Cleanup(func() { certMirrorInterval = prev })
+ log.SetOutput(io.Discard)
+ t.Cleanup(func() { log.SetOutput(os.Stderr) })
+
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour)))
+ m := newCertMirror(s, local)
+ m.issuing = func() bool { return true }
+ m.allowed = func(string) bool { return true }
+ stop := make(chan struct{})
+ done := make(chan struct{})
+ go func() { m.run(stop); close(done) }()
+ deadline := time.Now().Add(2 * time.Second)
+ for {
+ if _, ok := bucketHas(t, s, "a.example.com"); ok {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("the loop never backfilled")
+ }
+ time.Sleep(2 * time.Millisecond)
+ }
+ // And the mirror then serves what it published.
+ deadline = time.Now().Add(2 * time.Second)
+ for {
+ if c, ok := m.Cert("a.example.com"); ok && c != nil {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("the mirror never served the backfilled cert")
+ }
+ time.Sleep(2 * time.Millisecond)
+ }
+ close(stop)
+ <-done
+}
+
+// The bucket side is part of the stamp: an entry settled as in sync that
+// later leaves the bucket (a prune, a mis-delete) republishes on the next
+// pass even though the file has not moved — otherwise #291's failure mode
+// returns by a different door.
+func TestBackfillRepublishesAnEntryThatLeftTheBucket(t *testing.T) {
+ log.SetOutput(io.Discard)
+ t.Cleanup(func() { log.SetOutput(os.Stderr) })
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour)))
+ ctx := context.Background()
+ m := issuerMirror(t, s, local, true)
+ m.backfill(ctx)
+ if err := m.Refresh(ctx); err != nil {
+ t.Fatal(err)
+ }
+ m.backfill(ctx) // settles in sync
+ if err := s.Delete(ctx, certKey("a.example.com")); err != nil {
+ t.Fatal(err)
+ }
+ if err := m.Refresh(ctx); err != nil {
+ t.Fatal(err)
+ }
+ m.backfill(ctx)
+ if _, ok := bucketHas(t, s, "a.example.com"); !ok {
+ t.Fatal("an entry that left the bucket was never republished")
+ }
+}
+
+// The production wiring: the wildcard manager's parent set reaches the
+// mirror, or every wildcard PEM would silently never backfill.
+func TestAttachWildcardToMirrorWiresParentsAndWake(t *testing.T) {
+ w := &wildcardManager{parents: []string{"acct.example.com"}}
+ m := newCertMirror(release.NewDirStore(t.TempDir()), nil)
+ attachWildcardToMirror(m, w)
+ if m.wildcardParents == nil || !slices.Equal(m.wildcardParents(), []string{"acct.example.com"}) {
+ t.Fatal("wildcardParents not wired to the manager's current set")
+ }
+ if m.wake == nil {
+ t.Fatal("wake not wired")
+ }
+ attachWildcardToMirror(nil, w) // no mirror: no panic
+ attachWildcardToMirror(m, nil) // no manager: leaves the wiring alone
+ if m.wildcardParents == nil {
+ t.Fatal("a nil manager cleared the wiring")
+ }
+}
diff --git a/internal/edge/certsync.go b/internal/edge/certsync.go
index 06eaeabc3..dcb41069a 100644
--- a/internal/edge/certsync.go
+++ b/internal/edge/certsync.go
@@ -39,6 +39,8 @@ import (
"net/http"
"os"
"path"
+ "path/filepath"
+ "slices"
"strings"
"sync"
"time"
@@ -423,6 +425,21 @@ type certMirror struct {
// material on every poll.
local autocert.Cache
+ // issuing reports whether this box holds the issuer gate RIGHT NOW — the
+ // backfill's switch (see backfill). nil = never the issuer, which is the
+ // safe default for a mirror built without an Edge.
+ issuing func() bool
+ // wildcardParents is the wildcard manager's current parent set, the
+ // bound on which wildcard PEMs backfill may publish. nil = no manager =
+ // none.
+ wildcardParents func() []string
+ // settled memoes the stamp each on-disk entry was last found settled at
+ // (backfill); backfillFailures counts failed passes for the log
+ // cadence; backfillOnce guards the one-time "cannot walk" line.
+ settled map[string]fileStamp
+ backfillFailures int
+ backfillOnce sync.Once
+
// wake is called after a poll that changed wildcard material — the
// wildcard manager's refresh (R4), so a takeover propagates on the poll
// instead of on the manager's own 12h tick. nil when wildcards are off.
@@ -505,9 +522,31 @@ func (e *Edge) newCertMirror(local autocert.Cache) *certMirror {
// exists for a name that passed it — now that the policy admits the
// deployment's remote-homed routes too (#277).
m.allowed = e.deploymentHost
+ // Strict: an ungated box (no agent, e.issuer nil) is "held" for serving
+ // but is nobody's issuer, and must not publish its disk.
+ m.issuing = func() bool {
+ if e.issuer == nil {
+ return false
+ }
+ _, held := e.issuer()
+ return held
+ }
return m
}
+// attachWildcardToMirror is the mirror↔wildcard-manager wiring, in one place
+// so a test can see it: a wildcard the ISSUER minted reaches this box's
+// handshakes on the next poll rather than on the manager's 12h tick (wake),
+// and the manager's current parent set bounds which wildcard PEMs the
+// issuer's backfill may publish. Either side nil is a no-op.
+func attachWildcardToMirror(m *certMirror, w *wildcardManager) {
+ if m == nil || w == nil {
+ return
+ }
+ m.wake = w.refresh
+ m.wildcardParents = w.currentParents
+}
+
func newCertMirror(s release.Store, local autocert.Cache) *certMirror {
return &certMirror{
s: s,
@@ -742,7 +781,228 @@ func (m *certMirror) run(stop <-chan struct{}) {
log.Printf("edge: cert mirror refresh recovered after %d failure(s)", errs)
errs = 0
}
+ ctx, cancel = context.WithTimeout(context.Background(), certMirrorTimeout)
+ m.backfill(ctx)
+ cancel()
+ }
+}
+
+// backfill publishes what this box's cache holds and the bucket does not —
+// the issuer's half of "every edge serves every cert" for material minted
+// BEFORE publishing existed (#291).
+//
+// publishingCache publishes on Put, which is to say on issuance and renewal.
+// A deployment that arms a second edge after a year of single-box service
+// therefore starts with a bucket holding nothing but what was minted since
+// the gate went in: the flagship roll found 4 objects against 50 live
+// certs, and every handshake on a new region edge failed until renewal —
+// 30 to 60 days out. Renewal is the wrong repair for that; this is a
+// one-pass copy, run after every successful poll, that converges in one
+// tick. Steady state on the issuer is one directory listing per poll plus
+// a stat per entry: an entry whose size and mtime have not moved since it
+// was last settled — in sync with the bucket, or found not to be this
+// deployment's — is not re-read (m.settled).
+//
+// Rules, each doing work:
+//
+// - Only while holding the issuer gate — the STRICT gate (m.issuing), not
+// the serving-side "held or ungated" read: a hand-run `carlos edge`
+// with a bucket but no agent is ungated for serving and must NOT dump
+// its disk into the deployment's prefix. The gate is re-read per entry:
+// a pass over fifty certs can outlive a holding, and the writer map
+// gives control/certs/** to the housekeeper alone. Losing it ends the
+// pass but still reports what the pass did.
+// - Only a DirCache is walked. The mirror holds the cache as an
+// autocert.Cache, and that interface cannot enumerate; the on-disk
+// layout is the only backend that exists (certcache.go) and the one
+// whose entries ARE plain files named by cache key.
+// - Only names this deployment owns publish. The disk of a long-lived box
+// holds a cert for every route ever detached or renamed (autocert never
+// deletes), and DirCache.Put leaves "<name><random>" temp files behind
+// a kill mid-write — neither is the deployment's to replicate onto every
+// edge. A per-host entry (<host>, <host>+rsa) must pass m.allowed, the
+// same registry-backed exact check both read-throughs use; a wildcard
+// PEM must be for a parent in the wildcard manager's current set and
+// its leaf must cover it. Account keys are their own shape.
+// - Challenge tokens never publish: they are seconds long, and a copy of
+// a spent one is litter every edge would go on serving.
+// - Account keys publish only when the bucket has none. The deployment's
+// ACME identity is whichever key was published first (the mirror seeds
+// it onto every box, and a takeover issuer orders under it); a box's
+// own older key must never displace it, or the deployment would swap
+// accounts on every failover.
+// - A cert-shaped entry publishes when the bucket lacks it, or holds an
+// older leaf (NotAfter earlier than this box's) — the renewed-before-
+// the-gate case. Anything that does not parse, or has already expired,
+// stays on disk: a stale rehearsal name is nothing to distribute, and
+// the expiry check is what keeps this from re-publishing it forever.
+// - Failures are per-entry, retried next poll, and logged on the file's
+// first-and-every-tenth cadence: a write-denied bucket must not say so
+// every 30 seconds forever.
+func (m *certMirror) backfill(ctx context.Context) {
+ if m.issuing == nil || !m.issuing() || m.s == nil {
+ return
+ }
+ dir, ok := m.local.(autocert.DirCache)
+ if !ok {
+ m.backfillOnce.Do(func() {
+ log.Printf("certsync: backfill off — the local cert cache is not an on-disk cache and cannot be walked; only certs minted from now on reach %s*", certPrefix)
+ })
+ return
+ }
+ entries, err := os.ReadDir(string(dir))
+ if err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ m.logBackfillFailure(0, err)
+ }
+ return
+ }
+ snap := m.snapshot()
+ now := time.Now()
+ var published, failed int
+ var lastErr error
+ for _, ent := range entries {
+ name := ent.Name()
+ if !ent.Type().IsRegular() || !publishableCertName(name) {
+ continue
+ }
+ if strings.HasSuffix(name, httpTokenSuffix) || strings.HasSuffix(name, alpnTokenSuffix) {
+ continue
+ }
+ remote, inBucket := snap.entries[name]
+ info, err := ent.Info()
+ if err != nil {
+ continue
+ }
+ stamp := fileStamp{size: info.Size(), mtime: info.ModTime().UnixNano(), inBucket: inBucket}
+ // Settled at this stamp on an earlier pass — in sync with the
+ // bucket, or not this deployment's to publish — and nothing has
+ // moved since (the bucket's side is part of the stamp: an entry
+ // that was in sync but has since left the bucket re-reads). This
+ // is what keeps steady state at a listing plus a stat per entry.
+ if m.settledAt(name) == stamp {
+ continue
+ }
+ if !m.issuing() {
+ break
+ }
+ body, err := os.ReadFile(filepath.Join(string(dir), name))
+ if err != nil {
+ failed++
+ lastErr = err
+ continue
+ }
+ if inBucket && bytes.Equal(remote, body) {
+ // Also the backstop for names the snapshot never parses: the
+ // freshness compare below cannot see those, and without this
+ // they would republish every poll.
+ m.markSettled(name, stamp)
+ continue
+ }
+ if accountKeyName(name) {
+ if inBucket {
+ m.markSettled(name, stamp)
+ continue
+ }
+ } else {
+ cert, err := parseCertEntry(body)
+ if err != nil || !now.Before(cert.Leaf.NotAfter) || !m.ownsCertName(name, cert) {
+ // Expiry is a function of time, but a settled entry can only
+ // move from live to expired, never back; and ownership can
+ // return (a route re-attached), which is a registry change —
+ // so an entry settled as not-owned is re-examined only when
+ // its stamp moves. A re-attached route mints afresh anyway.
+ m.markSettled(name, stamp)
+ continue
+ }
+ if inBucket {
+ if have := snap.certs[name]; have != nil && !have.Leaf.NotAfter.Before(cert.Leaf.NotAfter) {
+ m.markSettled(name, stamp)
+ continue
+ }
+ }
+ }
+ if err := putCert(ctx, m.s, name, body); err != nil {
+ failed++
+ lastErr = err
+ continue
+ }
+ // Not settled: the bucket side of the stamp changes on the next
+ // poll, and the bytes.Equal path settles it then.
+ published++
+ }
+ if published > 0 {
+ log.Printf("certsync: backfilled %d cache entr%s from this box into %s* — every edge serves them on its next poll", published, plural(published, "y", "ies"), certPrefix)
+ }
+ if failed > 0 {
+ m.logBackfillFailure(failed, lastErr)
+ }
+}
+
+// ownsCertName reports whether a cert-shaped cache entry is one this
+// deployment should replicate: a per-host entry for a name the registry
+// still serves (m.allowed — nil admits nothing, the safe default for a
+// mirror built without an Edge), or a wildcard PEM for a parent in the
+// wildcard manager's CURRENT parent set (m.wildcardParents — nil, no
+// manager, admits none) whose leaf covers that parent.
+func (m *certMirror) ownsCertName(name string, cert *tls.Certificate) bool {
+ if parent, ok := strings.CutPrefix(name, "wildcard-"); ok && strings.HasSuffix(parent, ".pem") {
+ parent = strings.TrimSuffix(parent, ".pem")
+ if parent == "" || m.wildcardParents == nil || !slices.Contains(m.wildcardParents(), parent) {
+ return false
+ }
+ return slices.Contains(cert.Leaf.DNSNames, "*."+parent)
+ }
+ host := strings.TrimSuffix(name, "+rsa")
+ if m.allowed == nil || !m.allowed(host) {
+ return false
+ }
+ return cert.Leaf.VerifyHostname(host) == nil
+}
+
+// fileStamp is the state a backfill pass last settled an entry at: the
+// file's size+mtime and whether the bucket held the name. Not a hash:
+// DirCache writes are atomic renames, so a changed body is a changed
+// stamp (nanosecond mtime, so a same-size rewrite within one second on a
+// coarse filesystem is the only alias, and autocert never rewrites a name
+// that fast).
+type fileStamp struct {
+ size int64
+ mtime int64
+ inBucket bool
+}
+
+func (m *certMirror) settledAt(name string) fileStamp {
+ m.mu.RLock()
+ defer m.mu.RUnlock()
+ return m.settled[name]
+}
+
+func (m *certMirror) markSettled(name string, st fileStamp) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if m.settled == nil {
+ m.settled = map[string]fileStamp{}
+ }
+ m.settled[name] = st
+}
+
+// logBackfillFailure is logPartialPoll's cadence for the publish side.
+func (m *certMirror) logBackfillFailure(failed int, err error) {
+ m.mu.Lock()
+ m.backfillFailures++
+ n := m.backfillFailures
+ m.mu.Unlock()
+ if n == 1 || n%10 == 0 {
+ log.Printf("certsync: backfill: %d entr%s not published (#%d, last: %v) — retried on the next poll", failed, plural(failed, "y", "ies"), n, err)
+ }
+}
+
+func plural(n int, one, many string) string {
+ if n == 1 {
+ return one
}
+ return many
}
// syncLocal copies mirrored material into this box's OWN autocert cache. It
diff --git a/internal/edge/edge.go b/internal/edge/edge.go
index 7b376d8ff..3e4e35d7a 100644
--- a/internal/edge/edge.go
+++ b/internal/edge/edge.go
@@ -2380,11 +2380,7 @@ func RunWithActivatorReady(args []string, act Activator, cache *Cache, ready fun
if w := newWildcardFromEnv(e, certCache); w != nil {
getCert = wrapGetCertificate(w, getCert)
e.OnReload(w.refresh)
- if mirror != nil {
- // A wildcard the ISSUER minted reaches this box's handshakes on the
- // next poll rather than on this manager's 12h tick.
- mirror.wake = w.refresh
- }
+ attachWildcardToMirror(mirror, w)
w.refresh()
go w.run()
log.Printf("edge: wildcard certs on (static %v, apps domain %q, delegation zone %q)",
diff --git a/internal/edge/wildcard.go b/internal/edge/wildcard.go
index 34d21e482..22bc0f261 100644
--- a/internal/edge/wildcard.go
+++ b/internal/edge/wildcard.go
@@ -227,6 +227,14 @@ func (w *wildcardManager) refresh() {
}
}
+// currentParents is the last computed parent set — the mirror's bound on
+// which wildcard PEMs the issuer's backfill may publish (certsync.go).
+func (w *wildcardManager) currentParents() []string {
+ w.mu.Lock()
+ defer w.mu.Unlock()
+ return append([]string(nil), w.parents...)
+}
+
// certFor returns the wildcard cert for serverName, or nil when the name
// isn't exactly one label under a current parent (a wildcard doesn't cover
// the apex or deeper levels) or no usable cert is loaded — nil sends the