carlos / platform Public

certsync: backfill runs on every agent-run box; only account-key seeding stays lease-gated (#318)

The rung-1 failover rehearsal found the issuer-only backfill rule left
every cert minted on a box that never held the housekeeper lease sitting
on that box's disk alone — exactly the bytes a failover loses with the
box, and the standby spent ~8 minutes handshake-dead because of it.

Cert entries in control/certs/** become multi-writer: any box run under
an agent's election backfills the entries it owns, kept convergent by
the existing freshness rules (bytes-equal settles; an older leaf never
displaces a newer one; expired or unowned entries never publish). Three
things deliberately do not widen: issuance (still lease-gated),
challenge tokens (never backfilled), and seeding the ACME account key —
N boxes racing an empty bucket would leave certs ordered under one
account beside another's stored key, so that stays with the strict gate,
and an unseeded key is deliberately not settled so the same box publishes
it on the first pass where it holds the lease. A hand-run carlos edge
without an agent still never publishes; the mid-pass lease re-check goes
with the rule that needed it.

Writer-map amendment: platform-infrastructure#26.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

Paul Campbell pushed by paul@keymail.dev 1 month ago eb8afa43a3691eb3e5797d05cf46aefd9dd7fe82 parent 65c6956

2 files changed, +104 −54

internal/edge/certbackfill_test.go +47 −31
internal/edge/certsync.go +57 −23

diff --git a/internal/edge/certbackfill_test.go b/internal/edge/certbackfill_test.go

index 30f7e6e6e..5fbc36e9c 100644

--- a/internal/edge/certbackfill_test.go

+++ b/internal/edge/certbackfill_test.go

@@ -23,6 +23,7 @@ import (

func issuerMirror(t *testing.T, s release.Store, local autocert.Cache, held bool) *certMirror {

t.Helper()

m := newCertMirror(s, local)

+ m.mayBackfill = func() bool { return true } // an agent-run box

m.issuing = func() bool { return held }

// The registry: every *.example.com name except "gone".

m.allowed = func(host string) bool {

@@ -128,31 +129,46 @@ func (c *putCountingStore) Put(ctx context.Context, key string, body []byte, ifM

return c.Store.Put(ctx, key, body, ifMatch)

}

-// A non-issuer's disk is never the truth: the writer map gives the prefix

-// to the housekeeper alone.

-func TestBackfillIsIssuerOnly(t *testing.T) {

+// #318: certs publish from ANY agent-run box, lease or no lease — the

+// rung-1 rehearsal found the issuer-only rule left every cert minted on a

+// box that never held the lease sitting on that box's disk alone, exactly

+// the bytes a failover loses. Only the agent-run predicate bars a mirror:

+// a hand-run edge, and a mirror built without an Edge, still never publish.

+func TestBackfillRunsOnANonIssuer(t *testing.T) {

+ log.SetOutput(io.Discard)

+ t.Cleanup(func() { log.SetOutput(os.Stderr) })

s := release.NewDirStore(t.TempDir())

local := autocert.DirCache(t.TempDir())

putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour)))

- m := issuerMirror(t, s, local, false)

- m.backfill(context.Background())

- if _, ok := bucketHas(t, s, "a.example.com"); ok {

- t.Fatal("a non-issuer published")

- }

- m2 := newCertMirror(s, local) // issuing nil: never the issuer

+ m2 := newCertMirror(s, local) // mayBackfill nil: no Edge, never publishes

m2.backfill(context.Background())

if _, ok := bucketHas(t, s, "a.example.com"); ok {

t.Fatal("a gate-less mirror published")

}

- // The Edge wiring is STRICT: a box with a store but no agent gate is

- // ungated for serving (issuerGate says held) yet is nobody's issuer.

+ m := issuerMirror(t, s, local, false) // agent-run, NOT holding the lease

+ m.backfill(context.Background())

+ if _, ok := bucketHas(t, s, "a.example.com"); !ok {

+ t.Fatal("an agent-run non-issuer did not publish its own cert")

+ }

+ // The Edge wiring: agent-run is e.issuer != nil, independent of held;

+ // a store-carrying edge with no agent (hand-run) may never backfill.

e := &Edge{staticStore: s}

m3 := e.newCertMirror(local)

+ if m3.mayBackfill() {

+ t.Fatal("a hand-run edge may backfill")

+ }

+ if m3.issuing() {

+ t.Fatal("an ungated edge counts as the issuer")

+ }

+ e.issuer = func() (context.Context, bool) { return context.Background(), false }

+ if !m3.mayBackfill() {

+ t.Fatal("an agent-run non-holder may not backfill")

+ }

if m3.issuing() {

- t.Fatal("an ungated edge counts as the issuer for backfill")

+ t.Fatal("a non-holding edge counts as the issuer")

}

e.issuer = func() (context.Context, bool) { return context.Background(), true }

if !m3.issuing() {

@@ -160,28 +176,27 @@ func TestBackfillIsIssuerOnly(t *testing.T) {

}

-// The gate is re-read per entry: a holding that ends mid-pass stops the

-// pass where it stands.

-func TestBackfillStopsWhenTheGateIsLostMidPass(t *testing.T) {

+// Seeding the deployment's ACME identity stays single-writer: a non-issuer

+// never offers its own key to an empty bucket — and the unseeded key is NOT

+// settled, so the same box publishes it on the first pass where it holds.

+func TestBackfillAccountKeySeedsOnlyFromTheIssuer(t *testing.T) {

+ log.SetOutput(io.Discard)

+ t.Cleanup(func() { log.SetOutput(os.Stderr) })

s := release.NewDirStore(t.TempDir())

local := autocert.DirCache(t.TempDir())

- later := time.Now().Add(time.Hour)

- for _, h := range []string{"a.example.com", "b.example.com", "c.example.com"} {

- putLocal(t, local, h, hostCertPEM(t, h, later))

- }

- var buf strings.Builder

- log.SetOutput(&buf)

- t.Cleanup(func() { log.SetOutput(os.Stderr) })

- m := issuerMirror(t, s, local, true)

- calls := 0

- m.issuing = func() bool { calls++; return calls <= 2 } // the top check, then one entry

+ putLocal(t, local, "acme_account+key", []byte("MINE"))

+ held := false

+ m := issuerMirror(t, s, local, false)

+ m.issuing = func() bool { return held }

m.backfill(context.Background())

- keys, _ := s.List(context.Background(), certPrefix)

- if len(keys) != 1 {

- t.Fatalf("published %v after the gate was lost; want exactly one", keys)

+ if _, ok := bucketHas(t, s, "acme_account+key"); ok {

+ t.Fatal("a non-issuer seeded the account key")

}

- if !strings.Contains(buf.String(), "backfilled 1 cache entry") {

- t.Fatalf("a pass cut short by the gate did not report what it published:\n%s", buf.String())

+ held = true // the same process later wins the lease: same stamps, same memo

+ m.backfill(context.Background())

+ if got, _ := bucketHas(t, s, "acme_account+key"); string(got) != "MINE" {

+ t.Fatalf("the new issuer did not seed the account key it holds: %q", got)

}

@@ -427,6 +442,7 @@ func TestCertMirrorLoopBackfillsOnTheIssuer(t *testing.T) {

local := autocert.DirCache(t.TempDir())

putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour)))

m := newCertMirror(s, local)

+ m.mayBackfill = func() bool { return true }

m.issuing = func() bool { return true }

m.allowed = func(string) bool { return true }

stop := make(chan struct{})

diff --git a/internal/edge/certsync.go b/internal/edge/certsync.go

index dcb41069a..0e24c8297 100644

--- a/internal/edge/certsync.go

+++ b/internal/edge/certsync.go

@@ -425,9 +425,16 @@ type certMirror struct {

// material on every poll.

local autocert.Cache

- // issuing reports whether this box holds the issuer gate RIGHT NOW — the

- // backfill's switch (see backfill). nil = never the issuer, which is the

- // safe default for a mirror built without an Edge.

+ // mayBackfill reports whether this box may publish its disk's certs at

+ // all — the backfill's switch (see backfill): true on a box run under an

+ // agent (whose election makes it an accountable member of the

+ // deployment), false on a hand-run `carlos edge` and nil — never — for a

+ // mirror built without an Edge.

+ mayBackfill func() bool

+ // issuing reports whether this box holds the issuer gate RIGHT NOW —

+ // what remains single-writer inside backfill: seeding the ACME account

+ // key (#318 widened cert publishing to every agent-run box). nil = never

+ // the issuer, the safe default for a mirror built without an Edge.

issuing func() bool

// wildcardParents is the wildcard manager's current parent set, the

// bound on which wildcard PEMs backfill may publish. nil = no manager =

@@ -522,8 +529,15 @@ func (e *Edge) newCertMirror(local autocert.Cache) *certMirror {

// exists for a name that passed it — now that the policy admits the

// deployment's remote-homed routes too (#277).

m.allowed = e.deploymentHost

- // Strict: an ungated box (no agent, e.issuer nil) is "held" for serving

- // but is nobody's issuer, and must not publish its disk.

+ // Backfill runs on any AGENT-run box (#318): the election in the process

+ // is what makes the box an accountable member of the deployment, whether

+ // or not it currently holds the lease. A hand-run `carlos edge` with a

+ // bucket but no agent is "held" for serving and must still NOT dump its

+ // disk into the deployment's prefix — that is this predicate, not the

+ // lease.

+ m.mayBackfill = func() bool { return e.issuer != nil }

+ // Strict: only the box holding the lease RIGHT NOW may seed the

+ // deployment's ACME account identity (see backfill's account-key rule).

m.issuing = func() bool {

if e.issuer == nil {

return false

@@ -788,8 +802,8 @@ func (m *certMirror) run(stop <-chan struct{}) {

}

// backfill publishes what this box's cache holds and the bucket does not —

-// the issuer's half of "every edge serves every cert" for material minted

-// BEFORE publishing existed (#291).

+// every box's half of "every edge serves every cert" for material minted

+// BEFORE publishing existed (#291), on whichever box's disk it sits (#318).

// publishingCache publishes on Put, which is to say on issuance and renewal.

// A deployment that arms a second edge after a year of single-box service

@@ -805,13 +819,22 @@ func (m *certMirror) run(stop <-chan struct{}) {

// Rules, each doing work:

-// - Only while holding the issuer gate — the STRICT gate (m.issuing), not

-// the serving-side "held or ungated" read: a hand-run `carlos edge`

-// with a bucket but no agent is ungated for serving and must NOT dump

-// its disk into the deployment's prefix. The gate is re-read per entry:

-// a pass over fifty certs can outlive a holding, and the writer map

-// gives control/certs/** to the housekeeper alone. Losing it ends the

-// pass but still reports what the pass did.

+// - Only on an agent-run box (m.mayBackfill), NOT only on the issuer.

+// The 2026-08-27 rung-1 rehearsal (#318) found the issuer-only rule

+// left every cert minted on a box that never held the lease since

+// #291 sitting on that box's disk alone — exactly the bytes a

+// failover loses with the box. Cert entries in control/certs/** are

+// therefore multi-writer now, kept convergent by the freshness rules

+// below (bytes-equal settles; an older leaf never displaces a newer

+// one), and the writer map's housekeeper-only line for this prefix is

+// amended to "any agent-run box, certs only". The serving-side "held

+// or ungated" read is still the wrong predicate: a hand-run `carlos

+// edge` with a bucket but no agent is ungated for serving and must

+// NOT dump its disk into the deployment's prefix — m.mayBackfill is

+// false exactly there. One consequence to hold in mind when a cert

+// PURGE ever lands: any box's mirrored on-disk copy now resurrects a

+// deleted bucket entry on its next pass, so a purge must tombstone or

+// it will not stick.

// - Only a DirCache is walked. The mirror holds the cache as an

// autocert.Cache, and that interface cannot enumerate; the on-disk

// layout is the only backend that exists (certcache.go) and the one

@@ -826,11 +849,19 @@ func (m *certMirror) run(stop <-chan struct{}) {

// its leaf must cover it. Account keys are their own shape.

// - Challenge tokens never publish: they are seconds long, and a copy of

// a spent one is litter every edge would go on serving.

-// - Account keys publish only when the bucket has none. The deployment's

-// ACME identity is whichever key was published first (the mirror seeds

-// it onto every box, and a takeover issuer orders under it); a box's

-// own older key must never displace it, or the deployment would swap

-// accounts on every failover.

+// - Account keys publish only when the bucket has none, and only from

+// the box holding the STRICT issuer gate (m.issuing) — the one

+// backfill decision that stays single-writer. The deployment's ACME

+// identity is whichever key was published first (the mirror seeds it

+// onto every box, and a takeover issuer orders under it); a box's own

+// older key must never displace it, or the deployment would swap

+// accounts on every failover — and N agent boxes racing an EMPTY

+// bucket with their own keys would leave certs ordered under one

+// account stored beside another's key, which is why the widened cert

+// rule above does not extend here. A non-issuer's unseeded key is

+// deliberately NOT settled: the box may hold the lease on a later

+// pass, and a settled stamp would sit on exactly the pass that could

+// finally publish it.

// - A cert-shaped entry publishes when the bucket lacks it, or holds an

// older leaf (NotAfter earlier than this box's) — the renewed-before-

// the-gate case. Anything that does not parse, or has already expired,

@@ -840,7 +871,7 @@ func (m *certMirror) run(stop <-chan struct{}) {

// first-and-every-tenth cadence: a write-denied bucket must not say so

// every 30 seconds forever.

func (m *certMirror) backfill(ctx context.Context) {

- if m.issuing == nil || !m.issuing() || m.s == nil {

+ if m.mayBackfill == nil || !m.mayBackfill() || m.s == nil {

return

}

dir, ok := m.local.(autocert.DirCache)

@@ -883,9 +914,6 @@ func (m *certMirror) backfill(ctx context.Context) {

if m.settledAt(name) == stamp {

continue

}

- if !m.issuing() {

- break

- }

body, err := os.ReadFile(filepath.Join(string(dir), name))

if err != nil {

failed++

@@ -904,6 +932,12 @@ func (m *certMirror) backfill(ctx context.Context) {

m.markSettled(name, stamp)

continue

}

+ if m.issuing == nil || !m.issuing() {

+ // Only the elected issuer seeds the deployment's ACME

+ // identity (the rule above). Not settled, deliberately —

+ // see the same rule for why.

+ continue

+ }

} else {

cert, err := parseCertEntry(body)

if err != nil || !now.Before(cert.Leaf.NotAfter) || !m.ownsCertName(name, cert) {