amadan

carlos / platform Public

certsync: the issuer backfills pre-existing certs into control/certs/ (#291)

Publish-on-Put only ever covered material minted after the gate armed.
A deployment arming its second edge after a year of single-box service
started with a near-empty bucket — the flagship roll found 4 objects
against ~50 live certs — and every region-edge handshake failed until
renewal, 30-60 days out.

The mirror now runs a backfill after each successful poll, on the box
holding the STRICT issuer gate only (an ungated hand-run edge never
publishes; the gate is re-read per entry): walk the on-disk autocert
cache, publish every live cert-shaped entry the deployment owns that
the bucket lacks or holds an older leaf for. Owned = a per-host entry
passing the registry's exact-host check, or a wildcard PEM covering
the parent it is named for — a long-lived disk holds every detached
route and the temp files a kill left behind, and none of that is the
deployment's to replicate. Account keys publish only when the bucket
has none (the shared ACME identity is whichever key landed first; a
takeover issuer's own key never displaces it). Challenge tokens,
expired and unparseable entries stay on disk. Converges in one tick;
steady state is a listing plus a stat per entry, and failures log on
the file's first-and-every-tenth cadence.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Paul Campbell pushed by paul@keymail.dev 17c6621349ed4e68317cbbf9278da8e09e60b3e4 parent d6cd0e9
3 files changed, +665 −0
  • docs/superpowers/specs/2026-08-20-multi-region-ingress-design.md +18 −0
  • internal/edge/certbackfill_test.go +428 −0
  • internal/edge/certsync.go +219 −0
diff --git a/docs/superpowers/specs/2026-08-20-multi-region-ingress-design.md b/docs/superpowers/specs/2026-08-20-multi-region-ingress-design.md
index 246e84bf6..530fcf771 100644
--- a/docs/superpowers/specs/2026-08-20-multi-region-ingress-design.md
+++ b/docs/superpowers/specs/2026-08-20-multi-region-ingress-design.md
@@ -147,6 +147,24 @@ local certs mirror on their own 30-second list-poll, and
`GetCertificate` serves from that mirror. Renewals run ~30 days early,
so a 30s propagation cadence is never on a deadline.
+*Amendment 2026-08-25 (#291).* Publish-on-Put alone left everything
+minted BEFORE the gate armed out of the bucket (the flagship roll found
+4 objects against ~50 live certs, and region edges served nothing until
+renewal). The issuer therefore also **backfills**: after every
+successful mirror poll, the box holding the gate walks its on-disk
+cache and publishes every live cert-shaped entry the bucket lacks or
+holds an older leaf for; account keys publish only when the bucket has
+none (the deployment's ACME identity is whichever key landed first, and
+a takeover issuer's own key never displaces it); challenge tokens,
+expired and unparseable entries never publish; nor does anything the
+deployment does not own — a per-host entry must pass the registry's
+exact-host check, a wildcard PEM must cover the parent it is named for
+(a long-lived box's disk holds every route ever detached, and DirCache
+temp files a kill left behind). The switch is the strict housekeeper
+gate, re-read per entry — an ungated hand-run edge never publishes.
+Converges in one tick; steady state on the issuer is one directory
+listing plus a stat per entry per poll.
+
**Keys in the bucket — the trust statement.** Any box with deployment-
bucket read already serves this deployment's traffic and reads its
releases; holding TLS keys does not extend what a compromised box can
diff --git a/internal/edge/certbackfill_test.go b/internal/edge/certbackfill_test.go
new file mode 100644
index 000000000..bddfe75fd
--- /dev/null
+++ b/internal/edge/certbackfill_test.go
@@ -0,0 +1,428 @@
+package edge
+
+import (
+ "context"
+ "errors"
+ "io"
+ "log"
+ "os"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "golang.org/x/crypto/acme/autocert"
+
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// issuerMirror is a mirror over local+bucket whose box holds the issuer gate
+// when held is true. Refresh has run, so the snapshot reflects the bucket.
+func issuerMirror(t *testing.T, s release.Store, local autocert.Cache, held bool) *certMirror {
+ t.Helper()
+ m := newCertMirror(s, local)
+ m.issuing = func() bool { return held }
+ // The registry: every *.example.com name except "gone".
+ m.allowed = func(host string) bool {
+ return strings.HasSuffix(host, ".example.com") && !strings.HasPrefix(host, "gone.")
+ }
+ if err := m.Refresh(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ return m
+}
+
+func putLocal(t *testing.T, c autocert.Cache, name string, body []byte) {
+ t.Helper()
+ if err := c.Put(context.Background(), name, body); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func bucketHas(t *testing.T, s release.Store, name string) ([]byte, bool) {
+ t.Helper()
+ b, err := certStoreGet(t, s, certKey(name))
+ if err != nil {
+ return nil, false
+ }
+ return b, true
+}
+
+// The flagship roll's shape: a cache full of certs minted before publishing
+// existed, a bucket holding almost nothing. One pass and the bucket has
+// every live cert — and nothing that should not be there.
+func TestBackfillPublishesPreExistingCertsOnce(t *testing.T) {
+ log.SetOutput(io.Discard)
+ t.Cleanup(func() { log.SetOutput(os.Stderr) })
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ later := time.Now().Add(60 * 24 * time.Hour)
+
+ live := hostCertPEM(t, "a.example.com", later)
+ putLocal(t, local, "a.example.com", live)
+ putLocal(t, local, "a.example.com+rsa", hostCertPEM(t, "a.example.com", later))
+ putLocal(t, local, "wildcard-acct.example.com.pem", hostCertPEM(t, "*.acct.example.com", later))
+ putLocal(t, local, "expired.example.com", hostCertPEM(t, "expired.example.com", time.Now().Add(-time.Hour)))
+ putLocal(t, local, "garbage.example.com", []byte("not a cert"))
+ putLocal(t, local, "a.example.com+http-01", []byte("token-body"))
+ putLocal(t, local, "a.example.com+token", hostCertPEM(t, "a.example.com", later))
+ // Disk residue that is NOT the deployment's to replicate: a route
+ // detached long ago, a DirCache temp file a kill left behind, a
+ // wildcard PEM named for a parent its leaf does not cover.
+ putLocal(t, local, "gone.example.com", hostCertPEM(t, "gone.example.com", later))
+ putLocal(t, local, "a.example.com1762384410", live)
+ putLocal(t, local, "wildcard-other.example.com.pem", hostCertPEM(t, "*.acct.example.com", later))
+ putLocal(t, local, "acme_account+key", []byte("LOCAL-ACCOUNT"))
+ putLocal(t, local, wildcardAccountKeyName, []byte("LOCAL-WILDCARD-ACCOUNT"))
+
+ m := issuerMirror(t, s, local, true)
+ m.backfill(context.Background())
+
+ for _, name := range []string{"a.example.com", "a.example.com+rsa", "wildcard-acct.example.com.pem"} {
+ if _, ok := bucketHas(t, s, name); !ok {
+ t.Errorf("%s not backfilled", name)
+ }
+ }
+ if got, _ := bucketHas(t, s, "a.example.com"); string(got) != string(live) {
+ t.Errorf("a.example.com body differs from the local cache")
+ }
+ for _, name := range []string{"expired.example.com", "garbage.example.com", "a.example.com+http-01", "a.example.com+token",
+ "gone.example.com", "a.example.com1762384410", "wildcard-other.example.com.pem"} {
+ if _, ok := bucketHas(t, s, name); ok {
+ t.Errorf("%s must not be published", name)
+ }
+ }
+ // Account keys: the bucket had none, so this box's become the
+ // deployment's.
+ for name, want := range map[string]string{"acme_account+key": "LOCAL-ACCOUNT", wildcardAccountKeyName: "LOCAL-WILDCARD-ACCOUNT"} {
+ if got, ok := bucketHas(t, s, name); !ok || string(got) != want {
+ t.Errorf("%s: got %q, %v; want seeded", name, got, ok)
+ }
+ }
+
+ // Steady state: a second pass after a poll writes nothing. A fresh
+ // store wrapper counts.
+ if err := m.Refresh(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ c := &putCountingStore{Store: s}
+ m.s = c
+ m.backfill(context.Background())
+ if c.puts != 0 {
+ t.Fatalf("second pass published %d entries; want 0", c.puts)
+ }
+}
+
+type putCountingStore struct {
+ release.Store
+ puts int
+}
+
+func (c *putCountingStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) {
+ c.puts++
+ return c.Store.Put(ctx, key, body, ifMatch)
+}
+
+// A non-issuer's disk is never the truth: the writer map gives the prefix
+// to the housekeeper alone.
+func TestBackfillIsIssuerOnly(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour)))
+
+ m := issuerMirror(t, s, local, false)
+ m.backfill(context.Background())
+ if _, ok := bucketHas(t, s, "a.example.com"); ok {
+ t.Fatal("a non-issuer published")
+ }
+
+ m2 := newCertMirror(s, local) // issuing nil: never the issuer
+ m2.backfill(context.Background())
+ if _, ok := bucketHas(t, s, "a.example.com"); ok {
+ t.Fatal("a gate-less mirror published")
+ }
+
+ // The Edge wiring is STRICT: a box with a store but no agent gate is
+ // ungated for serving (issuerGate says held) yet is nobody's issuer.
+ e := &Edge{staticStore: s}
+ m3 := e.newCertMirror(local)
+ if m3.issuing() {
+ t.Fatal("an ungated edge counts as the issuer for backfill")
+ }
+ e.issuer = func() (context.Context, bool) { return context.Background(), true }
+ if !m3.issuing() {
+ t.Fatal("a holding edge is not the issuer")
+ }
+}
+
+// The gate is re-read per entry: a holding that ends mid-pass stops the
+// pass where it stands.
+func TestBackfillStopsWhenTheGateIsLostMidPass(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ later := time.Now().Add(time.Hour)
+ for _, h := range []string{"a.example.com", "b.example.com", "c.example.com"} {
+ putLocal(t, local, h, hostCertPEM(t, h, later))
+ }
+ m := issuerMirror(t, s, local, true)
+ calls := 0
+ m.issuing = func() bool { calls++; return calls <= 2 } // the top check, then one entry
+ m.backfill(context.Background())
+ keys, _ := s.List(context.Background(), certPrefix)
+ if len(keys) != 1 {
+ t.Fatalf("published %v after the gate was lost; want exactly one", keys)
+ }
+}
+
+// The other cert shapes ride the same freshness path as plain hosts.
+func TestBackfillWildcardAndRSAThroughTheFreshnessPath(t *testing.T) {
+ log.SetOutput(io.Discard)
+ t.Cleanup(func() { log.SetOutput(os.Stderr) })
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ soon, later := time.Now().Add(24*time.Hour), time.Now().Add(48*time.Hour)
+
+ putBucket(t, s, "wildcard-acct.example.com.pem", hostCertPEM(t, "*.acct.example.com", soon))
+ newWild := hostCertPEM(t, "*.acct.example.com", later)
+ putLocal(t, local, "wildcard-acct.example.com.pem", newWild)
+
+ putBucket(t, s, "a.example.com+rsa", hostCertPEM(t, "a.example.com", soon))
+ newRSA := hostCertPEM(t, "a.example.com", later)
+ putLocal(t, local, "a.example.com+rsa", newRSA)
+
+ // A bucket object that does not parse is replaced by a live local one.
+ putBucket(t, s, "b.example.com", []byte("corrupt"))
+ goodB := hostCertPEM(t, "b.example.com", later)
+ putLocal(t, local, "b.example.com", goodB)
+
+ m := issuerMirror(t, s, local, true)
+ m.backfill(context.Background())
+ for name, want := range map[string][]byte{"wildcard-acct.example.com.pem": newWild, "a.example.com+rsa": newRSA, "b.example.com": goodB} {
+ if got, _ := bucketHas(t, s, name); string(got) != string(want) {
+ t.Errorf("%s: bucket copy not brought forward", name)
+ }
+ }
+}
+
+type failingPutStore struct {
+ release.Store
+ mu sync.Mutex
+ puts int
+}
+
+func (f *failingPutStore) Put(context.Context, string, []byte, string) (string, error) {
+ f.mu.Lock()
+ defer f.mu.Unlock()
+ f.puts++
+ return "", errors.New("AccessDenied")
+}
+
+// A write-denied bucket retries every pass and says so on the file's
+// first-and-every-tenth cadence, never every 30 seconds.
+func TestBackfillFailureIsRetriedAndThrottled(t *testing.T) {
+ var buf strings.Builder
+ log.SetOutput(&buf)
+ t.Cleanup(func() { log.SetOutput(os.Stderr) })
+ inner := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour)))
+ m := issuerMirror(t, inner, local, true)
+ f := &failingPutStore{Store: inner}
+ m.s = f
+ for i := 0; i < 12; i++ {
+ m.backfill(context.Background())
+ }
+ if f.puts != 12 {
+ t.Fatalf("puts = %d; want one retry per pass", f.puts)
+ }
+ if n := strings.Count(buf.String(), "not published"); n != 2 {
+ t.Fatalf("logged %d times over 12 failing passes; want 2 (#1, #10):\n%s", n, buf.String())
+ }
+}
+
+// Steady state does not re-read the disk: an entry whose stamp has not
+// moved since it matched the bucket is skipped on the listing alone.
+func TestBackfillSteadyStateDoesNotReread(t *testing.T) {
+ log.SetOutput(io.Discard)
+ t.Cleanup(func() { log.SetOutput(os.Stderr) })
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour)))
+ m := issuerMirror(t, s, local, true)
+ m.backfill(context.Background())
+ if err := m.Refresh(context.Background()); err != nil {
+ t.Fatal(err)
+ }
+ if err := os.Chmod(string(local), 0o100); err != nil { // list only, no reads
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { _ = os.Chmod(string(local), 0o700) })
+ if os.Getuid() == 0 {
+ t.Skip("root ignores mode bits")
+ }
+ c := &putCountingStore{Store: s}
+ m.s = c
+ m.backfill(context.Background())
+ if c.puts != 0 {
+ t.Fatalf("steady state published %d", c.puts)
+ }
+}
+
+// autocert writes the cache while the backfill walks it. Whatever the
+// interleaving, nothing but complete, owned entries reach the bucket.
+func TestBackfillRacesAConcurrentIssuance(t *testing.T) {
+ log.SetOutput(io.Discard)
+ t.Cleanup(func() { log.SetOutput(os.Stderr) })
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ later := time.Now().Add(time.Hour)
+ pems := map[string][]byte{}
+ for i := 0; i < 8; i++ {
+ h := string(rune('a'+i)) + ".example.com"
+ pems[h] = hostCertPEM(t, h, later)
+ }
+ m := issuerMirror(t, s, local, true)
+ var wg sync.WaitGroup
+ wg.Add(2)
+ go func() {
+ defer wg.Done()
+ for h, pem := range pems {
+ putLocal(t, local, h, pem)
+ }
+ }()
+ go func() {
+ defer wg.Done()
+ for i := 0; i < 20; i++ {
+ m.backfill(context.Background())
+ }
+ }()
+ wg.Wait()
+ m.backfill(context.Background())
+ keys, _ := s.List(context.Background(), certPrefix)
+ for _, k := range keys {
+ name := strings.TrimPrefix(k, certPrefix)
+ want, ok := pems[name]
+ if !ok {
+ t.Errorf("stray bucket key %s", k)
+ continue
+ }
+ if got, _ := bucketHas(t, s, name); string(got) != string(want) {
+ t.Errorf("%s: bucket body is not the cache's", name)
+ }
+ }
+ if len(keys) != len(pems) {
+ t.Fatalf("bucket has %d of %d", len(keys), len(pems))
+ }
+}
+
+// The renewed-before-the-gate case: the bucket's copy is older than the
+// disk's, so the disk wins; and the reverse — the bucket is fresher — leaves
+// the bucket alone (the mirror will bring the disk up instead).
+func TestBackfillReplacesOnlyAnOlderBucketCert(t *testing.T) {
+ log.SetOutput(io.Discard)
+ t.Cleanup(func() { log.SetOutput(os.Stderr) })
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ soon := time.Now().Add(24 * time.Hour)
+ later := time.Now().Add(48 * time.Hour)
+
+ oldPEM := hostCertPEM(t, "old.example.com", soon)
+ newPEM := hostCertPEM(t, "old.example.com", later)
+ putBucket(t, s, "old.example.com", oldPEM)
+ putLocal(t, local, "old.example.com", newPEM)
+
+ freshBucket := hostCertPEM(t, "fresh.example.com", later)
+ putBucket(t, s, "fresh.example.com", freshBucket)
+ putLocal(t, local, "fresh.example.com", hostCertPEM(t, "fresh.example.com", soon))
+
+ m := issuerMirror(t, s, local, true)
+ m.backfill(context.Background())
+
+ if got, _ := bucketHas(t, s, "old.example.com"); string(got) != string(newPEM) {
+ t.Error("an older bucket cert was not replaced by the disk's newer one")
+ }
+ if got, _ := bucketHas(t, s, "fresh.example.com"); string(got) != string(freshBucket) {
+ t.Error("a fresher bucket cert was clobbered by an older local one")
+ }
+}
+
+// The deployment's ACME identity is whichever key the bucket already
+// holds — a takeover issuer's own key must never displace it.
+func TestBackfillNeverReplacesABucketAccountKey(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ putBucket(t, s, "acme_account+key", []byte("DEPLOYMENT"))
+ putBucket(t, s, wildcardAccountKeyName, []byte("DEPLOYMENT-WILDCARD"))
+ // Refresh seeds the bucket keys locally only when absent; make the local
+ // ones differ first.
+ putLocal(t, local, "acme_account+key", []byte("MINE"))
+ putLocal(t, local, wildcardAccountKeyName, []byte("MINE-WILDCARD"))
+
+ m := issuerMirror(t, s, local, true)
+ m.backfill(context.Background())
+ if got, _ := bucketHas(t, s, "acme_account+key"); string(got) != "DEPLOYMENT" {
+ t.Fatalf("account key replaced: %q", got)
+ }
+ if got, _ := bucketHas(t, s, wildcardAccountKeyName); string(got) != "DEPLOYMENT-WILDCARD" {
+ t.Fatalf("wildcard account key replaced: %q", got)
+ }
+}
+
+// Only an on-disk cache can be enumerated; anything else is a quiet no-op,
+// never a panic in the poll loop.
+func TestBackfillNeedsADirCache(t *testing.T) {
+ s := release.NewDirStore(t.TempDir())
+ m := issuerMirror(t, s, nil, true)
+ m.backfill(context.Background())
+ m2 := issuerMirror(t, s, newCountingCache(t), true)
+ m2.backfill(context.Background())
+ m3 := issuerMirror(t, s, autocert.DirCache(t.TempDir()+"/missing"), true)
+ m3.backfill(context.Background())
+ if keys, _ := s.List(context.Background(), certPrefix); len(keys) != 0 {
+ t.Fatalf("published %v", keys)
+ }
+}
+
+// The poll loop runs the backfill after each successful refresh: an issuer
+// that boots with a full cache and an empty bucket fills it without waiting
+// for a renewal.
+func TestCertMirrorLoopBackfillsOnTheIssuer(t *testing.T) {
+ prev := certMirrorInterval
+ certMirrorInterval = 5 * time.Millisecond
+ t.Cleanup(func() { certMirrorInterval = prev })
+ log.SetOutput(io.Discard)
+ t.Cleanup(func() { log.SetOutput(os.Stderr) })
+
+ s := release.NewDirStore(t.TempDir())
+ local := autocert.DirCache(t.TempDir())
+ putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour)))
+ m := newCertMirror(s, local)
+ m.issuing = func() bool { return true }
+ m.allowed = func(string) bool { return true }
+ stop := make(chan struct{})
+ done := make(chan struct{})
+ go func() { m.run(stop); close(done) }()
+ deadline := time.Now().Add(2 * time.Second)
+ for {
+ if _, ok := bucketHas(t, s, "a.example.com"); ok {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("the loop never backfilled")
+ }
+ time.Sleep(2 * time.Millisecond)
+ }
+ // And the mirror then serves what it published.
+ deadline = time.Now().Add(2 * time.Second)
+ for {
+ if c, ok := m.Cert("a.example.com"); ok && c != nil {
+ break
+ }
+ if time.Now().After(deadline) {
+ t.Fatal("the mirror never served the backfilled cert")
+ }
+ time.Sleep(2 * time.Millisecond)
+ }
+ close(stop)
+ <-done
+}
diff --git a/internal/edge/certsync.go b/internal/edge/certsync.go
index 06eaeabc3..0d8303b5c 100644
--- a/internal/edge/certsync.go
+++ b/internal/edge/certsync.go
@@ -39,6 +39,8 @@ import (
"net/http"
"os"
"path"
+ "path/filepath"
+ "slices"
"strings"
"sync"
"time"
@@ -423,6 +425,17 @@ type certMirror struct {
// material on every poll.
local autocert.Cache
+ // issuing reports whether this box holds the issuer gate RIGHT NOW — the
+ // backfill's switch (see backfill). nil = never the issuer, which is the
+ // safe default for a mirror built without an Edge.
+ issuing func() bool
+ // inSync memoes the on-disk stamp each entry last matched the bucket
+ // at (backfill); backfillFailures counts failed passes for the log
+ // cadence; backfillOnce guards the one-time "cannot walk" line.
+ inSync map[string]fileStamp
+ backfillFailures int
+ backfillOnce sync.Once
+
// wake is called after a poll that changed wildcard material — the
// wildcard manager's refresh (R4), so a takeover propagates on the poll
// instead of on the manager's own 12h tick. nil when wildcards are off.
@@ -505,6 +518,15 @@ func (e *Edge) newCertMirror(local autocert.Cache) *certMirror {
// exists for a name that passed it — now that the policy admits the
// deployment's remote-homed routes too (#277).
m.allowed = e.deploymentHost
+ // Strict: an ungated box (no agent, e.issuer nil) is "held" for serving
+ // but is nobody's issuer, and must not publish its disk.
+ m.issuing = func() bool {
+ if e.issuer == nil {
+ return false
+ }
+ _, held := e.issuer()
+ return held
+ }
return m
}
@@ -742,7 +764,204 @@ func (m *certMirror) run(stop <-chan struct{}) {
log.Printf("edge: cert mirror refresh recovered after %d failure(s)", errs)
errs = 0
}
+ ctx, cancel = context.WithTimeout(context.Background(), certMirrorTimeout)
+ m.backfill(ctx)
+ cancel()
+ }
+}
+
+// backfill publishes what this box's cache holds and the bucket does not —
+// the issuer's half of "every edge serves every cert" for material minted
+// BEFORE publishing existed (#291).
+//
+// publishingCache publishes on Put, which is to say on issuance and renewal.
+// A deployment that arms a second edge after a year of single-box service
+// therefore starts with a bucket holding nothing but what was minted since
+// the gate went in: the flagship roll found 4 objects against 50 live
+// certs, and every handshake on a new region edge failed until renewal —
+// 30 to 60 days out. Renewal is the wrong repair for that; this is a
+// one-pass copy, run after every successful poll, that converges in one
+// tick. Steady state on the issuer is one directory listing per poll plus
+// a stat per entry: an entry whose size and mtime have not moved since it
+// was last seen in sync with the bucket is not re-read (m.inSync).
+//
+// Rules, each doing work:
+//
+// - Only while holding the issuer gate — the STRICT gate (m.issuing), not
+// the serving-side "held or ungated" read: a hand-run `carlos edge`
+// with a bucket but no agent is ungated for serving and must NOT dump
+// its disk into the deployment's prefix. The gate is re-read per entry:
+// a pass over fifty certs can outlive a holding, and the writer map
+// gives control/certs/** to the housekeeper alone.
+// - Only a DirCache is walked. The mirror holds the cache as an
+// autocert.Cache, and that interface cannot enumerate; the on-disk
+// layout is the only backend that exists (certcache.go) and the one
+// whose entries ARE plain files named by cache key.
+// - Only names this deployment owns publish. The disk of a long-lived box
+// holds a cert for every route ever detached or renamed (autocert never
+// deletes), and DirCache.Put leaves "<name><random>" temp files behind
+// a kill mid-write — neither is the deployment's to replicate onto every
+// edge. A per-host entry (<host>, <host>+rsa) must pass m.allowed, the
+// same registry-backed exact check both read-throughs use; a wildcard
+// PEM must be named for the parent its leaf actually covers. Account
+// keys are their own shape.
+// - Challenge tokens never publish: they are seconds long, and a copy of
+// a spent one is litter every edge would go on serving.
+// - Account keys publish only when the bucket has none. The deployment's
+// ACME identity is whichever key was published first (the mirror seeds
+// it onto every box, and a takeover issuer orders under it); a box's
+// own older key must never displace it, or the deployment would swap
+// accounts on every failover.
+// - A cert-shaped entry publishes when the bucket lacks it, or holds an
+// older leaf (NotAfter earlier than this box's) — the renewed-before-
+// the-gate case. Anything that does not parse, or has already expired,
+// stays on disk: a stale rehearsal name is nothing to distribute, and
+// the expiry check is what keeps this from re-publishing it forever.
+// - Failures are per-entry, retried next poll, and logged on the file's
+// first-and-every-tenth cadence: a write-denied bucket must not say so
+// every 30 seconds forever.
+func (m *certMirror) backfill(ctx context.Context) {
+ if m.issuing == nil || !m.issuing() || m.s == nil {
+ return
+ }
+ dir, ok := m.local.(autocert.DirCache)
+ if !ok {
+ m.backfillOnce.Do(func() {
+ log.Printf("certsync: backfill off — the local cert cache is not an on-disk cache and cannot be walked; only certs minted from now on reach %s*", certPrefix)
+ })
+ return
+ }
+ entries, err := os.ReadDir(string(dir))
+ if err != nil {
+ if !errors.Is(err, os.ErrNotExist) {
+ m.logBackfillFailure(1, err)
+ }
+ return
+ }
+ snap := m.snapshot()
+ now := time.Now()
+ var published, failed int
+ var lastErr error
+ for _, ent := range entries {
+ name := ent.Name()
+ if !ent.Type().IsRegular() || !publishableCertName(name) {
+ continue
+ }
+ if strings.HasSuffix(name, httpTokenSuffix) || strings.HasSuffix(name, alpnTokenSuffix) {
+ continue
+ }
+ remote, inBucket := snap.entries[name]
+ info, err := ent.Info()
+ if err != nil {
+ continue
+ }
+ stamp := fileStamp{size: info.Size(), mtime: info.ModTime()}
+ if inBucket && m.inSyncAt(name) == stamp {
+ continue
+ }
+ if !m.issuing() {
+ return
+ }
+ body, err := os.ReadFile(filepath.Join(string(dir), name))
+ if err != nil {
+ failed++
+ lastErr = err
+ continue
+ }
+ if inBucket && bytes.Equal(remote, body) {
+ // Also the backstop for names the snapshot never parses: the
+ // freshness compare below cannot see those, and without this
+ // they would republish every poll.
+ m.markInSync(name, stamp)
+ continue
+ }
+ if accountKeyName(name) {
+ if inBucket {
+ continue
+ }
+ } else {
+ cert, err := parseCertEntry(body)
+ if err != nil || !now.Before(cert.Leaf.NotAfter) || !m.ownsCertName(name, cert) {
+ continue
+ }
+ if inBucket {
+ if have := snap.certs[name]; have != nil && !have.Leaf.NotAfter.Before(cert.Leaf.NotAfter) {
+ continue
+ }
+ }
+ }
+ if err := putCert(ctx, m.s, name, body); err != nil {
+ failed++
+ lastErr = err
+ continue
+ }
+ m.markInSync(name, stamp)
+ published++
+ }
+ if published > 0 {
+ log.Printf("certsync: backfilled %d cache entr%s from this box into %s* — every edge serves them on its next poll", published, plural(published, "y", "ies"), certPrefix)
+ }
+ if failed > 0 {
+ m.logBackfillFailure(failed, lastErr)
+ }
+}
+
+// ownsCertName reports whether a cert-shaped cache entry is one this
+// deployment should replicate: a per-host entry for a name the registry
+// still serves (m.allowed — nil admits nothing, the safe default for a
+// mirror built without an Edge), or a wildcard PEM whose leaf covers
+// exactly the parent it is named for.
+func (m *certMirror) ownsCertName(name string, cert *tls.Certificate) bool {
+ if parent, ok := strings.CutPrefix(name, "wildcard-"); ok && strings.HasSuffix(parent, ".pem") {
+ parent = strings.TrimSuffix(parent, ".pem")
+ return parent != "" && slices.Contains(cert.Leaf.DNSNames, "*."+parent)
+ }
+ host := strings.TrimSuffix(name, "+rsa")
+ if m.allowed == nil || !m.allowed(host) {
+ return false
+ }
+ return cert.Leaf.VerifyHostname(host) == nil
+}
+
+// fileStamp is the size+mtime a backfill pass last saw an entry at while it
+// was in sync with the bucket. Not a hash: DirCache writes are atomic
+// renames, so a changed body is a changed stamp.
+type fileStamp struct {
+ size int64
+ mtime time.Time
+}
+
+func (m *certMirror) inSyncAt(name string) fileStamp {
+ m.mu.RLock()
+ defer m.mu.RUnlock()
+ return m.inSync[name]
+}
+
+func (m *certMirror) markInSync(name string, st fileStamp) {
+ m.mu.Lock()
+ defer m.mu.Unlock()
+ if m.inSync == nil {
+ m.inSync = map[string]fileStamp{}
+ }
+ m.inSync[name] = st
+}
+
+// logBackfillFailure is logPartialPoll's cadence for the publish side.
+func (m *certMirror) logBackfillFailure(failed int, err error) {
+ m.mu.Lock()
+ m.backfillFailures++
+ n := m.backfillFailures
+ m.mu.Unlock()
+ if n == 1 || n%10 == 0 {
+ log.Printf("certsync: backfill: %d entr%s not published (#%d, last: %v) — retried on the next poll", failed, plural(failed, "y", "ies"), n, err)
+ }
+}
+
+func plural(n int, one, many string) string {
+ if n == 1 {
+ return one
}
+ return many
}
// syncLocal copies mirrored material into this box's OWN autocert cache. It