| index dcb41069a..8dbca3f72 100644 |
| --- a/internal/edge/certsync.go |
| +++ b/internal/edge/certsync.go |
| @@ -30,6 +30,7 @@ package edge |
| import ( |
| "bytes" |
| "context" |
| + "crypto/sha256" |
| "crypto/tls" |
| "crypto/x509" |
| "errors" |
| @@ -124,7 +125,10 @@ var errNotIssuer = errors.New("acme: not available here") |
| // directly, and the door is the agent's client. A grandfathered fleet box |
| // that still carries the bucket env therefore reads control/certs/ over |
| // its own IAM, and is gated AND mirrored like any other non-issuer — a |
| -// transition state, resolved by the rollout dropping that env; |
| +// transition state, resolved by the rollout dropping that env. It also |
| +// never BACKFILLS (#318): mayBackfill requires contending in the |
| +// election, and a fleet box's housekeeper mode is "never" — its disk |
| +// holds a customer's certs, not this deployment's; |
| // - boxes with neither CARLOS_DEPLOYMENT_BUCKET nor CARLOS_DEPLOYMENT_DIR; |
| // - boxes with a bucket but no CARLOS_RELEASE_PUBKEY, where |
| // enableStaticFromEnv leaves staticStore nil. |
| @@ -269,11 +273,12 @@ func (p *publishingCache) Put(ctx context.Context, key string, data []byte) erro |
| // serving. |
| // |
| // It is gated like Put, and that is a deliberate trade: an issuer that |
| -// stands down mid-order leaves its token behind in the bucket, because the |
| -// writer map gives control/certs/** to the housekeeper alone and a box that |
| -// is no longer the housekeeper does not get to delete there. The spec's |
| -// answer to that litter is a TTL stamp swept from the mirror side, not a |
| -// second writer. |
| +// stands down mid-order leaves its token behind in the bucket, because |
| +// ISSUANCE artifacts stay the housekeeper's to write and un-write (#318 |
| +// widened only the backfill of settled cert entries, never tokens), and a |
| +// box that is no longer the housekeeper does not get to delete there. The |
| +// spec's answer to that litter is a TTL stamp swept from the mirror side, |
| +// not a second writer. |
| func (p *publishingCache) Delete(ctx context.Context, key string) error { |
| err := p.inner.Delete(ctx, key) |
| if !p.publishes(key) { |
| @@ -334,11 +339,15 @@ func publishableCertName(name string) bool { |
| // same reason (release.Store.Put with an empty ifMatch is create-only, so a |
| // renewal would otherwise lose to its own previous cert). |
| // |
| -// Last-writer-wins is the DESIGN here, not a compromise: one gated issuer |
| -// means there is normally no second writer at all, and the transient |
| -// double-holder the housekeeper's poll window allows writes an equally valid |
| -// cert for the same name. Whichever lands last is servable, and the next |
| -// renewal converges the pair. |
| +// Last-writer-wins is the DESIGN here, not a compromise. On the issuance |
| +// path, the transient double-holder the housekeeper's poll window allows |
| +// writes an equally valid cert for the same name, and whichever lands last |
| +// is servable. On the backfill path (#318: any contending box), a writer |
| +// racing a stale view can land an OLDER leaf — repaired, not prevented: |
| +// fileStamp's bucket-identity half un-settles the box holding the newer |
| +// one, whose next pass republishes it. A freshness check inside this |
| +// function would still race (check-then-put), so the repair loop is the |
| +// guarantee, and it converges because the NotAfter compare is monotone. |
| func putCert(ctx context.Context, s release.Store, name string, body []byte) error { |
| key := certKey(name) |
| for attempt := 0; attempt < 3; attempt++ { |
| @@ -361,9 +370,10 @@ func putCert(ctx context.Context, s release.Store, name string, body []byte) err |
| // ---- the mirror: every edge serves every cert -------------------------------- |
| // |
| // certMirror is the serving half of the pair at the top of this file, and the |
| -// reason the issuance gate above is safe to shut. It POLLS control/certs/ (it |
| -// never writes there — the writer map gives that prefix to the housekeeper |
| -// alone) and keeps the whole prefix in memory, so any edge can answer a |
| +// reason the issuance gate above is safe to shut. It POLLS control/certs/ |
| +// (its serving side never writes there; the one writer it carries is the |
| +// backfill below, bounded to cert entries this box owns — #318) and keeps |
| +// the whole prefix in memory, so any edge can answer a |
| // handshake for a name some OTHER edge issued, and can answer an HTTP-01 |
| // challenge for an order some OTHER edge is running. |
| // |
| @@ -425,9 +435,16 @@ type certMirror struct { |
| // material on every poll. |
| local autocert.Cache |
| |
| - // issuing reports whether this box holds the issuer gate RIGHT NOW — the |
| - // backfill's switch (see backfill). nil = never the issuer, which is the |
| - // safe default for a mirror built without an Edge. |
| + // mayBackfill reports whether this box may publish its disk's certs at |
| + // all — the backfill's switch (see backfill): true on a box run under an |
| + // agent (whose election makes it an accountable member of the |
| + // deployment), false on a hand-run `carlos edge` and nil — never — for a |
| + // mirror built without an Edge. |
| + mayBackfill func() bool |
| + // issuing reports whether this box holds the issuer gate RIGHT NOW — |
| + // what remains single-writer inside backfill: seeding the ACME account |
| + // key (#318 widened cert publishing to every agent-run box). nil = never |
| + // the issuer, the safe default for a mirror built without an Edge. |
| issuing func() bool |
| // wildcardParents is the wildcard manager's current parent set, the |
| // bound on which wildcard PEMs backfill may publish. nil = no manager = |
| @@ -522,8 +539,21 @@ func (e *Edge) newCertMirror(local autocert.Cache) *certMirror { |
| // exists for a name that passed it — now that the policy admits the |
| // deployment's remote-homed routes too (#277). |
| m.allowed = e.deploymentHost |
| - // Strict: an ungated box (no agent, e.issuer nil) is "held" for serving |
| - // but is nobody's issuer, and must not publish its disk. |
| + // Backfill runs on any box that CONTENDS in the deployment's issuer |
| + // election (#318): contending is what makes the box an accountable |
| + // member of the deployment, whether or not it currently holds the |
| + // lease. Two box classes are excluded, each for its own reason: a |
| + // hand-run `carlos edge` with a bucket but no agent is "held" for |
| + // serving and must NOT dump its disk into the deployment's prefix |
| + // (e.issuer nil, and no contender predicate was ever handed down); and |
| + // a grandfathered FLEET box still carrying the bucket env is agent-run |
| + // with a gate func, but its housekeeper mode is "never" — its disk |
| + // holds a customer's certs, which are not this deployment's to |
| + // replicate onto every platform edge (WithIssuerContender, and the |
| + // armCertIssuance doc's grandfathered-fleet-box paragraph). |
| + m.mayBackfill = func() bool { return e.issuer != nil && e.contends != nil && e.contends() } |
| + // Strict: only the box holding the lease RIGHT NOW may seed the |
| + // deployment's ACME account identity (see backfill's account-key rule). |
| m.issuing = func() bool { |
| if e.issuer == nil { |
| return false |
| @@ -788,8 +818,8 @@ func (m *certMirror) run(stop <-chan struct{}) { |
| } |
| |
| // backfill publishes what this box's cache holds and the bucket does not — |
| -// the issuer's half of "every edge serves every cert" for material minted |
| -// BEFORE publishing existed (#291). |
| +// every box's half of "every edge serves every cert" for material minted |
| +// BEFORE publishing existed (#291), on whichever box's disk it sits (#318). |
| // |
| // publishingCache publishes on Put, which is to say on issuance and renewal. |
| // A deployment that arms a second edge after a year of single-box service |
| @@ -805,13 +835,27 @@ func (m *certMirror) run(stop <-chan struct{}) { |
| // |
| // Rules, each doing work: |
| // |
| -// - Only while holding the issuer gate — the STRICT gate (m.issuing), not |
| -// the serving-side "held or ungated" read: a hand-run `carlos edge` |
| -// with a bucket but no agent is ungated for serving and must NOT dump |
| -// its disk into the deployment's prefix. The gate is re-read per entry: |
| -// a pass over fifty certs can outlive a holding, and the writer map |
| -// gives control/certs/** to the housekeeper alone. Losing it ends the |
| -// pass but still reports what the pass did. |
| +// - Only on a box that contends in the election, NOT only on the issuer |
| +// (m.mayBackfill: agent-run AND contending in the issuer election). |
| +// The 2026-08-27 rung-1 rehearsal (#318) found the issuer-only rule |
| +// left every cert minted on a box that never held the lease since |
| +// #291 sitting on that box's disk alone — exactly the bytes a |
| +// failover loses with the box. Cert entries in control/certs/** are |
| +// therefore multi-writer now, kept convergent by the freshness rules |
| +// below (bytes-equal settles; an older leaf never displaces a newer |
| +// one), and the writer map's housekeeper-only line for this prefix is |
| +// amended to "any contending box, certs only". A concurrent writer |
| +// racing a stale view CAN transiently land an older leaf over a newer |
| +// one; the identity half of fileStamp is what makes that transient — |
| +// the box holding the newer leaf un-settles on the changed bucket sum |
| +// and republishes next pass. The serving-side "held |
| +// or ungated" read is still the wrong predicate: a hand-run `carlos |
| +// edge` with a bucket but no agent is ungated for serving and must |
| +// NOT dump its disk into the deployment's prefix — m.mayBackfill is |
| +// false exactly there. One consequence to hold in mind when a cert |
| +// PURGE ever lands: any box's mirrored on-disk copy now resurrects a |
| +// deleted bucket entry on its next pass, so a purge must tombstone or |
| +// it will not stick. |
| // - Only a DirCache is walked. The mirror holds the cache as an |
| // autocert.Cache, and that interface cannot enumerate; the on-disk |
| // layout is the only backend that exists (certcache.go) and the one |
| @@ -826,11 +870,19 @@ func (m *certMirror) run(stop <-chan struct{}) { |
| // its leaf must cover it. Account keys are their own shape. |
| // - Challenge tokens never publish: they are seconds long, and a copy of |
| // a spent one is litter every edge would go on serving. |
| -// - Account keys publish only when the bucket has none. The deployment's |
| -// ACME identity is whichever key was published first (the mirror seeds |
| -// it onto every box, and a takeover issuer orders under it); a box's |
| -// own older key must never displace it, or the deployment would swap |
| -// accounts on every failover. |
| +// - Account keys publish only when the bucket has none, and only from |
| +// the box holding the STRICT issuer gate (m.issuing) — the one |
| +// backfill decision that stays single-writer. The deployment's ACME |
| +// identity is whichever key was published first (the mirror seeds it |
| +// onto every box, and a takeover issuer orders under it); a box's own |
| +// older key must never displace it, or the deployment would swap |
| +// accounts on every failover — and N agent boxes racing an EMPTY |
| +// bucket with their own keys would leave certs ordered under one |
| +// account stored beside another's key, which is why the widened cert |
| +// rule above does not extend here. A non-issuer's unseeded key is |
| +// deliberately NOT settled: the box may hold the lease on a later |
| +// pass, and a settled stamp would sit on exactly the pass that could |
| +// finally publish it. |
| // - A cert-shaped entry publishes when the bucket lacks it, or holds an |
| // older leaf (NotAfter earlier than this box's) — the renewed-before- |
| // the-gate case. Anything that does not parse, or has already expired, |
| @@ -840,7 +892,7 @@ func (m *certMirror) run(stop <-chan struct{}) { |
| // first-and-every-tenth cadence: a write-denied bucket must not say so |
| // every 30 seconds forever. |
| func (m *certMirror) backfill(ctx context.Context) { |
| - if m.issuing == nil || !m.issuing() || m.s == nil { |
| + if m.mayBackfill == nil || !m.mayBackfill() || m.s == nil { |
| return |
| } |
| dir, ok := m.local.(autocert.DirCache) |
| @@ -874,18 +926,17 @@ func (m *certMirror) backfill(ctx context.Context) { |
| if err != nil { |
| continue |
| } |
| - stamp := fileStamp{size: info.Size(), mtime: info.ModTime().UnixNano(), inBucket: inBucket} |
| + stamp := fileStamp{size: info.Size(), mtime: info.ModTime().UnixNano(), bucketSum: bucketSum(remote, inBucket)} |
| // Settled at this stamp on an earlier pass — in sync with the |
| // bucket, or not this deployment's to publish — and nothing has |
| - // moved since (the bucket's side is part of the stamp: an entry |
| - // that was in sync but has since left the bucket re-reads). This |
| - // is what keeps steady state at a listing plus a stat per entry. |
| + // moved since. The bucket copy's IDENTITY is part of the stamp: an |
| + // entry that was in sync but has since left the bucket, or been |
| + // overwritten by another box, re-reads — that re-read is the #318 |
| + // repair path (see fileStamp). This is what keeps steady state at |
| + // a listing plus a stat per entry. |
| if m.settledAt(name) == stamp { |
| continue |
| } |
| - if !m.issuing() { |
| - break |
| - } |
| body, err := os.ReadFile(filepath.Join(string(dir), name)) |
| if err != nil { |
| failed++ |
| @@ -904,6 +955,12 @@ func (m *certMirror) backfill(ctx context.Context) { |
| m.markSettled(name, stamp) |
| continue |
| } |
| + if m.issuing == nil || !m.issuing() { |
| + // Only the elected issuer seeds the deployment's ACME |
| + // identity (the rule above). Not settled, deliberately — |
| + // see the same rule for why. |
| + continue |
| + } |
| } else { |
| cert, err := parseCertEntry(body) |
| if err != nil || !now.Before(cert.Leaf.NotAfter) || !m.ownsCertName(name, cert) { |
| @@ -961,15 +1018,33 @@ func (m *certMirror) ownsCertName(name string, cert *tls.Certificate) bool { |
| } |
| |
| // fileStamp is the state a backfill pass last settled an entry at: the |
| -// file's size+mtime and whether the bucket held the name. Not a hash: |
| -// DirCache writes are atomic renames, so a changed body is a changed |
| -// stamp (nanosecond mtime, so a same-size rewrite within one second on a |
| -// coarse filesystem is the only alias, and autocert never rewrites a name |
| -// that fast). |
| +// file's size+mtime, and the IDENTITY of the bucket's copy (a sha256 of |
| +// its bytes; zero = absent), not merely its presence. The local half is |
| +// not a hash: DirCache writes are atomic renames, so a changed body is a |
| +// changed stamp (nanosecond mtime, so a same-size rewrite within one |
| +// second on a coarse filesystem is the only alias, and autocert never |
| +// rewrites a name that fast). The bucket half MUST be identity: with |
| +// cert entries multi-writer (#318), another box racing a stale view can |
| +// land an older leaf over a newer one this box already settled against — |
| +// a presence bool would leave the settled box blind to the clobber |
| +// forever (its file unmoved, the name still "in bucket"), while the sum |
| +// changes with the clobber, un-settles exactly the box holding the newer |
| +// leaf, and its next pass republishes it. Convergence within a poll or |
| +// two, newest leaf wins. |
| type fileStamp struct { |
| - size int64 |
| - mtime int64 |
| - inBucket bool |
| + size int64 |
| + mtime int64 |
| + bucketSum [sha256.Size]byte |
| +} |
| + |
| +// bucketSum is fileStamp's bucket-identity half. The zero value means |
| +// "absent", which cannot collide with a real entry: nothing under |
| +// control/certs/ is ever zero bytes, and sha256 of any body is nonzero. |
| +func bucketSum(body []byte, inBucket bool) [sha256.Size]byte { |
| + if !inBucket { |
| + return [sha256.Size]byte{} |
| + } |
| + return sha256.Sum256(body) |
| } |
| |
| func (m *certMirror) settledAt(name string) fileStamp { |
amadan