| 1 | package edge |
| 2 | |
| 3 | // Issuer-side certificate publishing (spec: |
| 4 | // docs/superpowers/specs/2026-08-20-multi-region-ingress-design.md, Part 2). |
| 5 | // |
| 6 | // With more than one edge in a deployment, exactly ONE of them talks to the |
| 7 | // CA — the housekeeper (internal/host/housekeeper.go) — and every artifact |
| 8 | // that issuance produces is published to the control bucket under |
| 9 | // control/certs/ so every OTHER edge can serve it. Two halves live here: |
| 10 | // |
| 11 | // - publishingCache wraps the autocert.Cache the whole issuance path |
| 12 | // already funnels through (certcache.go's seam, reserved for exactly |
| 13 | // this). autocert writes its ACME account key, every per-host cert+key |
| 14 | // PEM, and every HTTP-01 token response through its Cache — cluster |
| 15 | // support is that interface's documented purpose — and wildcard.go's |
| 16 | // DNS-01 manager shares the same handle. Wrapping it therefore captures |
| 17 | // per-host certs, wildcards, tokens and account keys with no autocert |
| 18 | // fork and no second abstraction. |
| 19 | // |
| 20 | // - the issuance gate: HostPolicy (per-host HTTP-01) and the wildcard |
| 21 | // manager's order path (DNS-01) both refuse to START work on a box that |
| 22 | // is not the housekeeper. Serving is never gated: an edge serves from |
| 23 | // its local cache and from the mirror of this prefix regardless. |
| 24 | // |
| 25 | // Cert writes never bump control/serial — the serial is the console's |
| 26 | // (writer map, Part 1); the mirror side polls this prefix on its own |
| 27 | // cadence. Renewals run ~30 days before expiry, so a poll cadence measured |
| 28 | // in seconds is never on a deadline. |
| 29 | |
| 30 | import ( |
| 31 | "bytes" |
| 32 | "context" |
| 33 | "crypto/sha256" |
| 34 | "crypto/tls" |
| 35 | "crypto/x509" |
| 36 | "errors" |
| 37 | "fmt" |
| 38 | "log" |
| 39 | "math/rand/v2" |
| 40 | "net" |
| 41 | "net/http" |
| 42 | "os" |
| 43 | "path" |
| 44 | "path/filepath" |
| 45 | "slices" |
| 46 | "strings" |
| 47 | "sync" |
| 48 | "time" |
| 49 | |
| 50 | "golang.org/x/crypto/acme" |
| 51 | "golang.org/x/crypto/acme/autocert" |
| 52 | "golang.org/x/net/idna" |
| 53 | |
| 54 | "github.com/carlosframework/platform/internal/lbhealth" |
| 55 | "github.com/carlosframework/platform/internal/release" |
| 56 | ) |
| 57 | |
| 58 | // certPrefix is where every issuance artifact lands in the control bucket. |
| 59 | // One prefix for the lot, named by the cache key autocert (or wildcard.go) |
| 60 | // chose, so the mirror can rebuild a whole cache from a single list — and so |
| 61 | // the fleet door's ACL has exactly one prefix to keep denying. |
| 62 | const certPrefix = "control/certs/" |
| 63 | |
| 64 | // certKey is the bucket key for cache entry name. The cache name goes |
| 65 | // through verbatim: "example.com", "example.com+rsa", "acme_account+key", |
| 66 | // "<token>+http-01", "wildcard-<parent>.pem", "wildcard-account.key". |
| 67 | func certKey(name string) string { return certPrefix + name } |
| 68 | |
| 69 | // IssuerGate answers "may this process start certificate issuance, and under |
| 70 | // what authority does that work run?" — the housekeeper's gate |
| 71 | // (internal/host), handed DOWN at edge construction because internal/host |
| 72 | // imports internal/edge and never the reverse. The context is cancelled the |
| 73 | // instant this box stops being the housekeeper, so an order in flight |
| 74 | // unwinds with the authority it started under. |
| 75 | // |
| 76 | // A nil gate means "ungated": `carlos edge` run without the agent, and every |
| 77 | // test, has no election in the process at all, and refusing issuance there |
| 78 | // would leave the box with no certificates rather than with today's |
| 79 | // behaviour. |
| 80 | type IssuerGate func() (context.Context, bool) |
| 81 | |
| 82 | // issuerGate is the Edge's read of its gate, with the ungated default |
| 83 | // applied. See the type's doc for why nil means "yes". |
| 84 | func (e *Edge) issuerGate() (context.Context, bool) { |
| 85 | if e.issuer == nil { |
| 86 | return context.Background(), true |
| 87 | } |
| 88 | return e.issuer() |
| 89 | } |
| 90 | |
| 91 | // errNotIssuer is what HostPolicy answers on a box that is not the |
| 92 | // deployment's certificate issuer. Sentinel (errors.Is-able) so the gate is |
| 93 | // distinguishable from "that host isn't mine", which is a different fault |
| 94 | // with a different fix. |
| 95 | // |
| 96 | // The message text is deliberately generic, not diagnostic: autocert's own |
| 97 | // HTTPHandler writes this exact string, verbatim, as the body of an |
| 98 | // unauthenticated :80 response (http.Error(w, err.Error(), 403) at |
| 99 | // autocert.go's HTTPHandler) — OUR code never gets a chance to intercept or |
| 100 | // reword it before it reaches the wire. "acme: this box is not the |
| 101 | // deployment's certificate issuer" told any anonymous prober both that |
| 102 | // multi-edge issuance exists and which box is not the issuer; the identity |
| 103 | // that matters everywhere else in this file is errors.Is(err, errNotIssuer), |
| 104 | // never the string, so softening it costs nothing internal. |
| 105 | var errNotIssuer = errors.New("acme: not available here") |
| 106 | |
| 107 | // armCertIssuance wires cert DISTRIBUTION and the issuance GATE together, |
| 108 | // because they are one decision and not two: a box may only be told "you are |
| 109 | // not the issuer" if some other box's certificates can reach it. |
| 110 | // |
| 111 | // WITH a deployment store: every issuance artifact publishes to |
| 112 | // control/certs/ and the gate applies — one elected issuer mints, every |
| 113 | // other edge serves the published copy. |
| 114 | // |
| 115 | // WITHOUT one there is nothing to publish to and, just as importantly, |
| 116 | // nothing to mirror FROM. The gate is cleared and the box issues for itself |
| 117 | // exactly as it always has. Three real box classes land here, and gating any |
| 118 | // of them would leave it permanently certificate-less — no new orders AND no |
| 119 | // serving out of its own on-disk cache, since autocert consults HostPolicy |
| 120 | // before its cache lookup: |
| 121 | // |
| 122 | // - fleet boxes (CARLOS_FLEET) carrying no deployment bucket of their own: |
| 123 | // the housekeeper never contends there, and there is nothing to mirror |
| 124 | // from. The fleet store DOOR is not what decides this and never was: the |
| 125 | // edge does not use the door at all — enableStaticFromEnv reads |
| 126 | // CARLOS_DEPLOYMENT_BUCKET / CARLOS_DEPLOYMENT_DIR and opens the store |
| 127 | // directly, and the door is the agent's client. A grandfathered fleet box |
| 128 | // that still carries the bucket env therefore reads control/certs/ over |
| 129 | // its own IAM, and is gated AND mirrored like any other non-issuer — a |
| 130 | // transition state, resolved by the rollout dropping that env. It also |
| 131 | // never BACKFILLS (#318): mayBackfill requires contending in the |
| 132 | // election, and a fleet box's housekeeper mode is "never" — its disk |
| 133 | // holds a customer's certs, not this deployment's; |
| 134 | // - boxes with neither CARLOS_DEPLOYMENT_BUCKET nor CARLOS_DEPLOYMENT_DIR; |
| 135 | // - boxes with a bucket but no CARLOS_RELEASE_PUBKEY, where |
| 136 | // enableStaticFromEnv leaves staticStore nil. |
| 137 | // |
| 138 | // Clearing the field (rather than only skipping the HostPolicy wrap) is what |
| 139 | // keeps the wildcard manager symmetric too: newWildcardFromEnv reads the |
| 140 | // same gate a few lines later. |
| 141 | func (e *Edge) armCertIssuance(inner autocert.Cache) autocert.Cache { |
| 142 | if e.staticStore == nil { |
| 143 | e.issuer = nil |
| 144 | log.Printf("edge: cert publishing off (no deployment store configured) — this box issues and keeps its own certs, as before") |
| 145 | return inner |
| 146 | } |
| 147 | log.Printf("edge: publishing issued certs to %s* in the deployment bucket", certPrefix) |
| 148 | return &publishingCache{inner: inner, s: e.staticStore, issuer: e.issuerGate} |
| 149 | } |
| 150 | |
| 151 | // issuerHostPolicy is the autocert HostPolicy: the registry check this edge |
| 152 | // has always applied, plus the issuance gate. |
| 153 | // |
| 154 | // The registry check is deploymentHost (edge.go) rather than Allowed: a |
| 155 | // deployment's issuer must be able to mint for any EXACTLY-named route in the |
| 156 | // deployment, not only the ones this box happens to home, or a home-boxed |
| 157 | // route whose hellos reach the issuer through the backhaul can never get a |
| 158 | // certificate at all (#277). Exact-only, so PR #180's rule is untouched: a |
| 159 | // label under a catch-all still drives no order. The errNotIssuer gate below |
| 160 | // is unaffected — non-issuers still order nothing, for any name. |
| 161 | // |
| 162 | // Two autocert call sites reach this, and both are deliberate: |
| 163 | // GetCertificate consults the policy BEFORE its cache lookup |
| 164 | // (autocert.go:292 in x/crypto v0.52.0), and HTTPHandler consults it before |
| 165 | // serving an HTTP-01 token (autocert.go:405). On a non-issuer both therefore |
| 166 | // decline — which is correct only because the mirror layer sits behind them |
| 167 | // and serves both certs and challenges from control/certs/. Nothing here may |
| 168 | // be relaxed into "allow if we have it cached": autocert treats an EXPIRED |
| 169 | // cached cert as a miss and would go straight to an order, so a policy that |
| 170 | // says yes for a cached name is a policy that lets a non-issuer order. |
| 171 | // |
| 172 | // It does NOT stop renewals, and nothing here should be read as claiming it |
| 173 | // does: autocert's renewal timers (renewal.go's domainRenewal.do) never |
| 174 | // consult HostPolicy. A timer started while this box held the lease keeps |
| 175 | // firing after a stand-down; each firing re-checks the cache first, and only |
| 176 | // orders when the cache has nothing fresh. Two consequences, both landing on |
| 177 | // Task 5: the mirror MUST write fresh certs into this box's LOCAL cache, so |
| 178 | // that cache-first check short-circuits the timer before it orders; and |
| 179 | // until it does, a stood-down box's timer fails HTTP-01 against our own |
| 180 | // gated handler every ~30 minutes, burning the shared account's |
| 181 | // failed-validation budget. (A tls-alpn-01 validation would even succeed — |
| 182 | // the token-cert branch at autocert.go:288 precedes the policy check — and |
| 183 | // then the gated publisher declines to publish the result.) |
| 184 | // |
| 185 | // The flag is not a fence. Under a hung lease renewal two boxes can |
| 186 | // transiently both believe they hold — the housekeeper's own stand-down is |
| 187 | // bounded by its 30s poll, and a lost renewal is only noticed on that poll. |
| 188 | // Brief double-issuance against the shared ACME account is inside the |
| 189 | // spec's accepted envelope (one account, one rate-limit bucket, renewals ~30 |
| 190 | // days early), so there is deliberately NO fencing token here and no |
| 191 | // readiness predicate on the election: a fence would trade a rare duplicate |
| 192 | // order for a deployment that can stop issuing altogether. |
| 193 | func (e *Edge) issuerHostPolicy() autocert.HostPolicy { |
| 194 | return func(_ context.Context, host string) error { |
| 195 | if !e.deploymentHost(host) { |
| 196 | return fmt.Errorf("acme: host %q not in registry", host) |
| 197 | } |
| 198 | if _, ok := e.issuerGate(); !ok { |
| 199 | return errNotIssuer |
| 200 | } |
| 201 | return nil |
| 202 | } |
| 203 | } |
| 204 | |
| 205 | // publishingCache is the autocert.Cache every issuance artifact goes |
| 206 | // through: the local cache first (durability, and autocert's own read path |
| 207 | // keeps working byte-for-byte), then a best-effort copy into the control |
| 208 | // bucket. |
| 209 | // |
| 210 | // Bucket failures are LOUD and never returned: a returned error aborts an |
| 211 | // issuance that has already succeeded locally, trading a cert this box could |
| 212 | // have served for a bucket blip. The mirror's next poll (and, failing that, |
| 213 | // the next renewal) re-publishes. |
| 214 | type publishingCache struct { |
| 215 | inner autocert.Cache |
| 216 | s release.Store |
| 217 | // issuer gates the BUCKET half only — the local cache is written |
| 218 | // unconditionally, because a non-issuer must still be able to serve what |
| 219 | // it was handed. nil = ungated (see IssuerGate). |
| 220 | // |
| 221 | // It is the gate's boolean that is used and not its context: publishing |
| 222 | // is not an order, and cancelling a publish because the holding ended |
| 223 | // mid-write would strand a freshly minted cert on one box's disk with |
| 224 | // nothing scheduled to re-publish it. The write runs on its own budget |
| 225 | // (publishCtx) instead. |
| 226 | issuer IssuerGate |
| 227 | } |
| 228 | |
| 229 | // certPublishTimeout budgets ONE bucket publish or delete. It exists because |
| 230 | // autocert calls this cache from under its own locks: deleteHTTPToken and |
| 231 | // deleteCertToken call Cache.Delete while HOLDING m.challengeMu for write, |
| 232 | // and both httpToken (the :80 challenge handler) and GetCertificate's |
| 233 | // tls-alpn-01 branch read under that same mutex. An unbudgeted bucket call |
| 234 | // there wedges challenge serving and ALPN handshakes on the issuer for as |
| 235 | // long as the network takes. The Put path is no better off inheriting |
| 236 | // autocert's own 5-10 minute order deadline. |
| 237 | // |
| 238 | // A var, not a const, so tests can shorten it. |
| 239 | var certPublishTimeout = 10 * time.Second |
| 240 | |
| 241 | // publishCtx budgets one bucket call, detached from the caller's |
| 242 | // cancellation. WithoutCancel is deliberate: an order that has already been |
| 243 | // abandoned still has to clean up its challenge token, and a cancelled |
| 244 | // context would skip exactly the delete that prevents litter every edge |
| 245 | // would go on to mirror and serve. |
| 246 | func publishCtx(ctx context.Context) (context.Context, context.CancelFunc) { |
| 247 | return context.WithTimeout(context.WithoutCancel(ctx), certPublishTimeout) |
| 248 | } |
| 249 | |
| 250 | func (p *publishingCache) Get(ctx context.Context, key string) ([]byte, error) { |
| 251 | return p.inner.Get(ctx, key) |
| 252 | } |
| 253 | |
| 254 | // Put writes locally, then publishes. Local first: if the disk write fails |
| 255 | // there is no cert, and publishing one the box itself cannot serve would be |
| 256 | // a lie in the bucket. |
| 257 | func (p *publishingCache) Put(ctx context.Context, key string, data []byte) error { |
| 258 | if err := p.inner.Put(ctx, key, data); err != nil { |
| 259 | return err |
| 260 | } |
| 261 | if !p.publishes(key) { |
| 262 | return nil |
| 263 | } |
| 264 | pctx, cancel := publishCtx(ctx) |
| 265 | defer cancel() |
| 266 | if err := putCert(pctx, p.s, key, data); err != nil { |
| 267 | log.Printf("certsync: bucket publish failed for %q: %v — the cert is live on this box; the issuer's next renewal republishes", key, err) |
| 268 | } |
| 269 | return nil |
| 270 | } |
| 271 | |
| 272 | // Delete removes both copies. The bucket copy goes even when the local |
| 273 | // delete fails: the entry autocert deletes is a spent HTTP-01 token, and a |
| 274 | // token left in a prefix every edge mirrors is litter every edge would keep |
| 275 | // serving. |
| 276 | // |
| 277 | // It is gated like Put, and that is a deliberate trade: an issuer that |
| 278 | // stands down mid-order leaves its token behind in the bucket, because |
| 279 | // ISSUANCE artifacts stay the housekeeper's to write and un-write (#318 |
| 280 | // widened only the backfill of settled cert entries, never tokens), and a |
| 281 | // box that is no longer the housekeeper does not get to delete there. The |
| 282 | // spec's answer to that litter is a TTL stamp swept from the mirror side, |
| 283 | // not a second writer. |
| 284 | func (p *publishingCache) Delete(ctx context.Context, key string) error { |
| 285 | err := p.inner.Delete(ctx, key) |
| 286 | if !p.publishes(key) { |
| 287 | return err |
| 288 | } |
| 289 | pctx, cancel := publishCtx(ctx) |
| 290 | defer cancel() |
| 291 | if derr := p.s.Delete(pctx, certKey(key)); derr != nil { |
| 292 | log.Printf("certsync: bucket delete failed for %q: %v — a spent challenge token may linger until it is overwritten", key, derr) |
| 293 | } |
| 294 | return err |
| 295 | } |
| 296 | |
| 297 | // publishes reports whether this box should write key to the bucket: only |
| 298 | // the issuer publishes, and only names that survive the round trip through a |
| 299 | // bucket key do. |
| 300 | func (p *publishingCache) publishes(key string) bool { |
| 301 | if p.s == nil { |
| 302 | return false |
| 303 | } |
| 304 | if p.issuer != nil { |
| 305 | if _, ok := p.issuer(); !ok { |
| 306 | // Not the issuer. Silent, like every other not-held path in this |
| 307 | // deployment: it is the permanent normal state of N-1 boxes. |
| 308 | return false |
| 309 | } |
| 310 | } |
| 311 | if !publishableCertName(key) { |
| 312 | log.Printf("certsync: refusing to publish cache entry %q — the name does not round-trip as a bucket key", key) |
| 313 | return false |
| 314 | } |
| 315 | return true |
| 316 | } |
| 317 | |
| 318 | // publishableCertName rejects cache names that would not survive as |
| 319 | // "control/certs/<name>": a "/" would put the entry outside the flat prefix |
| 320 | // the mirror maps back onto cache names, and a traversal segment has no |
| 321 | // business in a key derived from a hostname. Nothing autocert or wildcard.go |
| 322 | // generates fails this (autocert's HTTP-01 key is path.Base + "+http-01", |
| 323 | // and domains are punycode-validated before they reach a cache key); it is |
| 324 | // here so that a future key shape fails visibly rather than scattering |
| 325 | // objects through the control prefix. |
| 326 | func publishableCertName(name string) bool { |
| 327 | if name == "" || strings.Contains(name, "/") { |
| 328 | return false |
| 329 | } |
| 330 | for _, seg := range []string{"..", "\\"} { |
| 331 | if strings.Contains(name, seg) { |
| 332 | return false |
| 333 | } |
| 334 | } |
| 335 | return true |
| 336 | } |
| 337 | |
| 338 | // putCert writes key's body regardless of prior state: create-only when |
| 339 | // absent, etag-replace when present, re-reading on a lost race — the |
| 340 | // putReplacing idiom from internal/host's snapshot loop, needed here for the |
| 341 | // same reason (release.Store.Put with an empty ifMatch is create-only, so a |
| 342 | // renewal would otherwise lose to its own previous cert). |
| 343 | // |
| 344 | // Last-writer-wins is the DESIGN here, not a compromise. On the issuance |
| 345 | // path, the transient double-holder the housekeeper's poll window allows |
| 346 | // writes an equally valid cert for the same name, and whichever lands last |
| 347 | // is servable. On the backfill path (#318: any contending box), a writer |
| 348 | // racing a stale view can land an OLDER leaf — repaired, not prevented: |
| 349 | // fileStamp's bucket-identity half un-settles the box holding the newer |
| 350 | // one, whose next pass republishes it. A freshness check inside this |
| 351 | // function would still race (check-then-put), so the repair loop is the |
| 352 | // guarantee, and it converges because the NotAfter compare is monotone. |
| 353 | func putCert(ctx context.Context, s release.Store, name string, body []byte) error { |
| 354 | return putCAS(ctx, s, certKey(name), body) |
| 355 | } |
| 356 | |
| 357 | // putCAS is putCert's write, by key rather than by cert name — the same |
| 358 | // read-etag-then-If-Match loop, shared so the cert warm's outcome records |
| 359 | // (#299 §3e, certwarm.go) get the identical concurrency posture instead of a |
| 360 | // second copy of it that can drift. |
| 361 | func putCAS(ctx context.Context, s release.Store, key string, body []byte) error { |
| 362 | for attempt := 0; attempt < 3; attempt++ { |
| 363 | _, etag, err := s.Get(ctx, key) |
| 364 | switch { |
| 365 | case errors.Is(err, os.ErrNotExist): |
| 366 | etag = "" |
| 367 | case err != nil: |
| 368 | return fmt.Errorf("get %s: %w", key, err) |
| 369 | } |
| 370 | if _, err := s.Put(ctx, key, body, etag); err == nil { |
| 371 | return nil |
| 372 | } else if !errors.Is(err, release.ErrCASConflict) { |
| 373 | return fmt.Errorf("put %s: %w", key, err) |
| 374 | } |
| 375 | } |
| 376 | return fmt.Errorf("%s: lost CAS race three times", key) |
| 377 | } |
| 378 | |
| 379 | // ---- the mirror: every edge serves every cert -------------------------------- |
| 380 | // |
| 381 | // certMirror is the serving half of the pair at the top of this file, and the |
| 382 | // reason the issuance gate above is safe to shut. It POLLS control/certs/ |
| 383 | // (its serving side never writes there; the one writer it carries is the |
| 384 | // backfill below, bounded to cert entries this box owns — #318) and keeps |
| 385 | // the whole prefix in memory, so any edge can answer a |
| 386 | // handshake for a name some OTHER edge issued, and can answer an HTTP-01 |
| 387 | // challenge for an order some OTHER edge is running. |
| 388 | // |
| 389 | // Three consumers, in the order a request meets them: |
| 390 | // |
| 391 | // - GetCertificate (wrapGetCertificate, wildcard.go): the mirror is the |
| 392 | // PRIMARY serving path on a non-issuer, ahead of autocert, because |
| 393 | // autocert's HostPolicy runs BEFORE its cache lookup — a gated manager |
| 394 | // serves nothing at all, cached or not. |
| 395 | // - the :80 challenge handler (challengeHandler): in FRONT of autocert's |
| 396 | // HTTPHandler for the same reason — that handler 403s a non-issuer before |
| 397 | // it ever looks a token up. |
| 398 | // - the LOCAL autocert cache: every refresh copies fresh cert material into |
| 399 | // it. autocert's renewal timers never consult HostPolicy (see |
| 400 | // issuerHostPolicy), and renewal.go's do() checks the cache FIRST, so a |
| 401 | // mirror-refreshed local cache is what stops a stood-down box's stale |
| 402 | // timers from ordering — and what carries a box through the boot and |
| 403 | // crash windows where the election has not yet been decided. |
| 404 | // |
| 405 | // Account keys are the one thing that is mirrored but never served: they are |
| 406 | // issuer material and have no business in a map a handshake can reach. |
| 407 | |
| 408 | // certMirrorInterval is the poll cadence. Renewals run ~30 days before |
| 409 | // expiry, so nothing on this path is on a deadline — except live HTTP-01 |
| 410 | // validations, which the poll deliberately does NOT try to catch (see |
| 411 | // challengeResponse's read-through). |
| 412 | var certMirrorInterval = 30 * time.Second |
| 413 | |
| 414 | // certMirrorTimeout budgets one whole poll: list plus the gets. Generous, |
| 415 | // because it runs on its own goroutine and blocks nothing; bounded, because a |
| 416 | // stalled bucket must not wedge the loop forever. |
| 417 | var certMirrorTimeout = 2 * time.Minute |
| 418 | |
| 419 | // certMirrorBootTimeout budgets the ONE refresh that runs before the |
| 420 | // listeners bind. Much shorter than a poll's: a stalled bucket must delay the |
| 421 | // front door by seconds, not minutes, and the box still has its own cache to |
| 422 | // serve from while the loop catches up. |
| 423 | var certMirrorBootTimeout = 20 * time.Second |
| 424 | |
| 425 | // certRefreshConcurrency bounds the in-flight gets in Refresh. It is a |
| 426 | // latency fix, not a throughput one: the pass costs |
| 427 | // ceil(count/concurrency) x latency instead of count x latency, which at |
| 428 | // 101 certs and a 250ms round trip is about 2s instead of 25s. |
| 429 | // |
| 430 | // Ten, not sixteen, because the AWS SDK's default transport keeps ten idle |
| 431 | // connections per host (DefaultHTTPTransportMaxIdleConnsPerHost) and nothing |
| 432 | // here overrides it. Sixteen in flight means six connections closed at the |
| 433 | // end of every pass and re-handshaked on the next one, forever, on a 30s |
| 434 | // poll — about a dozen extra cross-region TLS handshakes a minute for one |
| 435 | // extra wave's worth of latency. |
| 436 | const certRefreshConcurrency = 10 |
| 437 | |
| 438 | // certSnapshot is one poll's result, swapped in whole. Immutable once |
| 439 | // published, so readers hold no lock beyond the pointer read. |
| 440 | type certSnapshot struct { |
| 441 | // entries is every mirrored name → body, account keys included (the local |
| 442 | // sync needs them; Cert refuses to serve them). |
| 443 | entries map[string][]byte |
| 444 | // certs is the parsed form of every cert-shaped entry, keyed by the same |
| 445 | // name. Parsing happens ONCE per poll per changed entry, never per |
| 446 | // handshake. A name whose body does not parse is absent, not nil. |
| 447 | certs map[string]*tls.Certificate |
| 448 | } |
| 449 | |
| 450 | var emptyCertSnapshot = &certSnapshot{entries: map[string][]byte{}, certs: map[string]*tls.Certificate{}} |
| 451 | |
| 452 | type certMirror struct { |
| 453 | s release.Store |
| 454 | // local is this box's OWN autocert cache — the raw one, NOT the |
| 455 | // publishing wrapper: a mirror write is a copy inwards, and pushing it |
| 456 | // back out to the bucket would have the issuer republishing its own |
| 457 | // material on every poll. |
| 458 | local autocert.Cache |
| 459 | |
| 460 | // mayBackfill reports whether this box may publish its disk's certs at |
| 461 | // all — the backfill's switch (see backfill): true on a box run under an |
| 462 | // agent (whose election makes it an accountable member of the |
| 463 | // deployment), false on a hand-run `carlos edge` and nil — never — for a |
| 464 | // mirror built without an Edge. |
| 465 | mayBackfill func() bool |
| 466 | // issuing reports whether this box holds the issuer gate RIGHT NOW — |
| 467 | // what remains single-writer inside backfill: seeding the ACME account |
| 468 | // key (#318 widened cert publishing to every agent-run box). nil = never |
| 469 | // the issuer, the safe default for a mirror built without an Edge. |
| 470 | issuing func() bool |
| 471 | // wildcardParents is the wildcard manager's current parent set, the |
| 472 | // bound on which wildcard PEMs backfill may publish. nil = no manager = |
| 473 | // none. |
| 474 | wildcardParents func() []string |
| 475 | // settled memoes the stamp each on-disk entry was last found settled at |
| 476 | // (backfill); backfillFailures counts failed passes for the log |
| 477 | // cadence; backfillOnce guards the one-time "cannot walk" line. |
| 478 | settled map[string]fileStamp |
| 479 | backfillFailures int |
| 480 | backfillOnce sync.Once |
| 481 | |
| 482 | // wake is called after a poll that changed wildcard material — the |
| 483 | // wildcard manager's refresh (R4), so a takeover propagates on the poll |
| 484 | // instead of on the manager's own 12h tick. nil when wildcards are off. |
| 485 | wake func() |
| 486 | |
| 487 | // allowed reports whether a hostname is one this box serves, or one |
| 488 | // another box in this deployment exactly homes (Edge.deploymentHost) — |
| 489 | // the bound on BOTH read-throughs, whose keys are attacker-chosen SNI and |
| 490 | // an attacker-chosen Host header. Exact names only, either side: a label |
| 491 | // under a catch-all spends neither budget (#180, and KnownHost's doc). |
| 492 | // nil = unbounded, which is tests only. |
| 493 | allowed func(string) bool |
| 494 | |
| 495 | // gated reports whether issuance is currently BARRED on this box, which |
| 496 | // is what decides where the mirror sits in the handshake chain (see |
| 497 | // wrapGetCertificate). Read per hello, because the election moves under a |
| 498 | // running process. nil = treat as gated, the safe default: it serves the |
| 499 | // same certificates either way, it only costs an issuer its lazily-armed |
| 500 | // renewal timers, and a mirror built without a gate (tests) has no |
| 501 | // issuance to arm. |
| 502 | gated func() bool |
| 503 | |
| 504 | // refreshMu serializes Refresh's whole body. Production has exactly one |
| 505 | // caller today (the 30s loop, run()), so this is insurance rather than a |
| 506 | // live fix: two overlapping polls (a second caller added later, or a |
| 507 | // boot-time refresh racing the loop's first tick) could otherwise both |
| 508 | // read m.snapshot() as prev, build independently, and publish out of |
| 509 | // order — the SECOND Store to finish under m.mu.Lock() wins regardless |
| 510 | // of which list was actually more recent, so a slow poll could clobber a |
| 511 | // fresher one. Locking the body makes Refresh calls queue and apply in |
| 512 | // the order they were made. |
| 513 | refreshMu sync.Mutex |
| 514 | |
| 515 | mu sync.RWMutex |
| 516 | snap *certSnapshot |
| 517 | // partialPolls counts polls that could not read every object, for the |
| 518 | // first-and-every-tenth log cadence. |
| 519 | partialPolls int |
| 520 | |
| 521 | // health is the load balancer health state the first landed refresh |
| 522 | // latches (region HA spec 2 §6); nil without region HA. |
| 523 | health *lbhealth.State |
| 524 | |
| 525 | // watch resolves the process's change-index watch (#336), handed down by |
| 526 | // the agent through WithChangeWatch and read on every poll: the agent |
| 527 | // arms its watch on its own schedule, possibly after the edge starts. |
| 528 | // nil, or a nil watch, is today's mirror: every poll a full Refresh. |
| 529 | // now is the poll's clock (nil is time.Now; tests inject one). |
| 530 | watch func() *release.ChangeWatch |
| 531 | now func() time.Time |
| 532 | // watchTimeout bounds the poll of the watch (0 is |
| 533 | // release.ChangeIndexPollTimeout; tests shorten it). |
| 534 | watchTimeout time.Duration |
| 535 | |
| 536 | // Guarded by refreshMu: what the published snapshot was read at. |
| 537 | // etags is each entry's ETag as its Get returned it ("" = unknown, read |
| 538 | // it again); lastFull is the last full rebuild that landed and fullAge |
| 539 | // the jittered backstop after which the next poll rebuilds in full; |
| 540 | // actedGen is the control/certs generation, captured BEFORE the read, |
| 541 | // that the snapshot is known to reflect (actedOK false = no such |
| 542 | // generation: the watch could not vouch, or some object went unread). |
| 543 | etags map[string]string |
| 544 | lastFull time.Time |
| 545 | fullAge time.Duration |
| 546 | actedGen uint64 |
| 547 | actedOK bool |
| 548 | // localPending: the last copy into the local cache left entries it |
| 549 | // could not write; an unchanged poll retries it from the snapshot. |
| 550 | localPending bool |
| 551 | |
| 552 | // synced remembers, per name, the mirrored bytes last reconciled into the |
| 553 | // local cache, so a steady state costs no local I/O at all. |
| 554 | syncMu sync.Mutex |
| 555 | synced map[string][]byte |
| 556 | |
| 557 | // rt is the read-through memo (hits and misses), its in-flight set, and |
| 558 | // the global token bucket that bounds live bucket reads. |
| 559 | rtMu sync.Mutex |
| 560 | rt map[string]readThroughEntry |
| 561 | rtInflight map[string]chan struct{} |
| 562 | rtHTTP rtBudget // :80, the HTTP-01 challenge path |
| 563 | rtALPN rtBudget // 443, the tls-alpn-01 token-cert path |
| 564 | lastRTLog time.Time |
| 565 | |
| 566 | // localCerts memoes the last-resort reads of this box's own cache; |
| 567 | // lastLocalLog throttles the line that says the mirror had nothing. |
| 568 | localMu sync.Mutex |
| 569 | localCerts map[string]localCertEntry |
| 570 | lastLocalLog time.Time |
| 571 | } |
| 572 | |
| 573 | // newCertMirror is the Edge's mirror, or nil when there is nothing to mirror |
| 574 | // FROM. The nil case is exactly armCertIssuance's no-store case — the two |
| 575 | // decisions are the same decision, and a box that lands there is ungated and |
| 576 | // serves its own cache through autocert as it always has. |
| 577 | func (e *Edge) newCertMirror(local autocert.Cache) *certMirror { |
| 578 | if e.staticStore == nil { |
| 579 | return nil |
| 580 | } |
| 581 | m := newCertMirror(e.staticStore, local) |
| 582 | m.health = e.lbHealth |
| 583 | m.watch = e.changeWatch |
| 584 | m.gated = func() bool { |
| 585 | _, held := e.issuerGate() |
| 586 | return !held |
| 587 | } |
| 588 | // Both read-throughs are keyed by remote input (SNI on 443, the Host |
| 589 | // header on :80), so they are bounded to the names this DEPLOYMENT serves: |
| 590 | // this box's own routes, plus the exact names another box's edges file |
| 591 | // homes. deploymentHost is the same exact-match, registry-backed check |
| 592 | // autocert's HostPolicy uses (issuerHostPolicy, above), which is the right |
| 593 | // bound for the same reason it always was — a per-host order only ever |
| 594 | // exists for a name that passed it — now that the policy admits the |
| 595 | // deployment's remote-homed routes too (#277). |
| 596 | m.allowed = e.deploymentHost |
| 597 | // Backfill runs on any box that CONTENDS in the deployment's issuer |
| 598 | // election (#318): contending is what makes the box an accountable |
| 599 | // member of the deployment, whether or not it currently holds the |
| 600 | // lease. Two box classes are excluded, each for its own reason: a |
| 601 | // hand-run `carlos edge` with a bucket but no agent is "held" for |
| 602 | // serving and must NOT dump its disk into the deployment's prefix |
| 603 | // (e.issuer nil, and no contender predicate was ever handed down); and |
| 604 | // a grandfathered FLEET box still carrying the bucket env is agent-run |
| 605 | // with a gate func, but its housekeeper mode is "never" — its disk |
| 606 | // holds a customer's certs, which are not this deployment's to |
| 607 | // replicate onto every platform edge (WithIssuerContender, and the |
| 608 | // armCertIssuance doc's grandfathered-fleet-box paragraph). |
| 609 | m.mayBackfill = func() bool { return e.issuer != nil && e.contends != nil && e.contends() } |
| 610 | // Strict: only the box holding the lease RIGHT NOW may seed the |
| 611 | // deployment's ACME account identity (see backfill's account-key rule). |
| 612 | m.issuing = func() bool { |
| 613 | if e.issuer == nil { |
| 614 | return false |
| 615 | } |
| 616 | _, held := e.issuer() |
| 617 | return held |
| 618 | } |
| 619 | return m |
| 620 | } |
| 621 | |
| 622 | // attachWildcardToMirror is the mirror↔wildcard-manager wiring, in one place |
| 623 | // so a test can see it: a wildcard the ISSUER minted reaches this box's |
| 624 | // handshakes on the next poll rather than on the manager's 12h tick (wake), |
| 625 | // and the manager's current parent set bounds which wildcard PEMs the |
| 626 | // issuer's backfill may publish. Either side nil is a no-op. |
| 627 | func attachWildcardToMirror(m *certMirror, w *wildcardManager) { |
| 628 | if m == nil || w == nil { |
| 629 | return |
| 630 | } |
| 631 | m.wake = w.refresh |
| 632 | m.wildcardParents = w.currentParents |
| 633 | } |
| 634 | |
| 635 | func newCertMirror(s release.Store, local autocert.Cache) *certMirror { |
| 636 | return &certMirror{ |
| 637 | s: s, |
| 638 | local: local, |
| 639 | snap: emptyCertSnapshot, |
| 640 | synced: map[string][]byte{}, |
| 641 | rt: map[string]readThroughEntry{}, |
| 642 | rtInflight: map[string]chan struct{}{}, |
| 643 | localCerts: map[string]localCertEntry{}, |
| 644 | } |
| 645 | } |
| 646 | |
| 647 | // Refresh rebuilds the whole map from one list plus a get per key. It is the |
| 648 | // boot read before the listeners bind, every poll while the change index is |
| 649 | // disarmed, and the armed loop's backstop and "unknown" read (poll, #336). |
| 650 | // A rebuild from the listing is what makes a withdrawn entry (a spent |
| 651 | // challenge token) actually leave; poll's changed-only read builds from a |
| 652 | // listing too, for the same reason. |
| 653 | // |
| 654 | // An error leaves the last-good snapshot in place and is returned for the |
| 655 | // caller to log. That is the edge's standing degrade-not-die posture, and it |
| 656 | // matters more here than usual: an empty map on a bucket blip would be a TLS |
| 657 | // outage on every non-issuer in the deployment. |
| 658 | func (m *certMirror) Refresh(ctx context.Context) error { |
| 659 | m.refreshMu.Lock() |
| 660 | defer m.refreshMu.Unlock() |
| 661 | return m.refreshFull(ctx) |
| 662 | } |
| 663 | |
| 664 | // refreshFull is Refresh's body, under refreshMu. |
| 665 | func (m *certMirror) refreshFull(ctx context.Context) error { |
| 666 | // The generation this read will reflect, captured BEFORE the list: a |
| 667 | // write that lands during the read leaves control/certs moved past it, |
| 668 | // so the next poll looks again (#336). Without a usable watch there is |
| 669 | // none, and the next poll cannot skip. |
| 670 | gen, genOK := m.certsGen() |
| 671 | keys, err := m.s.List(ctx, certPrefix) |
| 672 | if err != nil { |
| 673 | return fmt.Errorf("list %s: %w", certPrefix, err) |
| 674 | } |
| 675 | prev := m.snapshot() |
| 676 | next := make(map[string][]byte, len(keys)) |
| 677 | etags := make(map[string]string, len(keys)) |
| 678 | |
| 679 | // The eligible names first, then their bodies CONCURRENTLY. The fetch is |
| 680 | // one round trip per object and the objects are independent, so doing |
| 681 | // them one after another makes the whole pass cost |
| 682 | // count x latency — which is fine beside the bucket and is not fine |
| 683 | // across the planet. |
| 684 | // |
| 685 | // That is not hypothetical (amadan #118 thread, diagnosed 2026-09-19). |
| 686 | // The Sydney box reads this deployment's bucket in eu-west-1 at ~250ms a |
| 687 | // round trip. At 101 published certs the serial pass took ~25s, blowing |
| 688 | // certMirrorBootTimeout, and since this refresh runs BEFORE either |
| 689 | // listener binds, the edge was not serving until it finished. The |
| 690 | // self-update gate waits 20s for the edge to answer on loopback, so that |
| 691 | // box failed its gate, rolled back, failed the rollback's gate too, and |
| 692 | // took no update at all from 2026-09-07 to 2026-09-19. |
| 693 | // |
| 694 | // The count is what makes it a fleet problem rather than an AU one: every |
| 695 | // box pays count x latency, so each new published cert moves every box |
| 696 | // closer to the same cliff. us-east-1 at ~75ms is about a third of the |
| 697 | // way there at today's count. |
| 698 | var items []certFetch |
| 699 | for _, key := range keys { |
| 700 | name, ok := strings.CutPrefix(key, certPrefix) |
| 701 | // publishableCertName is the same filter the publish side applies: |
| 702 | // anything nested under the prefix was not written by this deployment's |
| 703 | // issuer and has no cache name to map back onto. |
| 704 | if !ok || !publishableCertName(name) { |
| 705 | continue |
| 706 | } |
| 707 | items = append(items, certFetch{key: key, name: name}) |
| 708 | } |
| 709 | tried := len(items) |
| 710 | failed, lastErr := applyCertFetches(items, m.fetchCerts(ctx, items), prev, next, etags) |
| 711 | if tried > 0 && failed == tried { |
| 712 | // Nothing at all could be read: that is a bucket fault, not an object |
| 713 | // fault, and it is reported like a failed List — last-good stands. |
| 714 | return fmt.Errorf("get %s*: every one of %d object(s) failed, last: %w", certPrefix, tried, lastErr) |
| 715 | } |
| 716 | if failed > 0 { |
| 717 | m.logPartialPoll(failed, tried, lastErr) |
| 718 | } |
| 719 | m.publish(ctx, prev, next, etags) |
| 720 | // A partial rebuild still counts as the full rebuild: the objects it |
| 721 | // could not read carry an unknown ETag, so the next poll's changed-only |
| 722 | // read fetches exactly those again. It reflects no generation, though, |
| 723 | // so that next poll cannot skip. |
| 724 | m.lastFull, m.fullAge = m.clock(), jitteredBackstop() |
| 725 | m.actedGen, m.actedOK = gen, genOK && failed == 0 |
| 726 | return nil |
| 727 | } |
| 728 | |
| 729 | // poll is the 30s loop's refresh (#336). Disarmed, it is Refresh. Armed, it |
| 730 | // polls the change index (a 304 when nothing moved, one GET either way) and |
| 731 | // then does the least that keeps the snapshot current: |
| 732 | // |
| 733 | // - nothing, when control/certs has not moved since the generation the |
| 734 | // snapshot reflects and the last full rebuild is younger than the |
| 735 | // jittered backstop; |
| 736 | // - a changed-only read (refreshChanged) when it has moved; |
| 737 | // - today's full Refresh when the watch cannot vouch for the scope (the |
| 738 | // index unreadable: "unknown", never "unchanged"), when no full rebuild |
| 739 | // has landed yet, or at the backstop. |
| 740 | // |
| 741 | // Poll's moved list is not consulted: the watch is shared with the agent's |
| 742 | // tick and record view, so whichever polls first consumes it. Generations |
| 743 | // are compared with the one this mirror last acted on instead. |
| 744 | func (m *certMirror) poll(ctx context.Context) error { |
| 745 | w := m.changeWatch() |
| 746 | if w == nil { |
| 747 | return m.Refresh(ctx) |
| 748 | } |
| 749 | // Bounded on its own: the watch is shared, and a hung index GET from |
| 750 | // another caller holds its poll lock. Any error, giving up on that lock |
| 751 | // included, is "unknown": a full read, as before the index. |
| 752 | _, perr := pollWatch(ctx, w, m.watchTimeout) |
| 753 | m.refreshMu.Lock() |
| 754 | defer m.refreshMu.Unlock() |
| 755 | gen, ok := w.Gen(certScope) |
| 756 | switch { |
| 757 | case perr != nil, !ok, m.lastFull.IsZero(), !m.clock().Before(m.lastFull.Add(m.fullAge)): |
| 758 | return m.refreshFull(ctx) |
| 759 | case m.actedOK && gen == m.actedGen: |
| 760 | // Nothing moved in the bucket, but a copy into the local cache |
| 761 | // that failed is retried from the snapshot already held: no |
| 762 | // bucket read (final review R1, finding 7). |
| 763 | if m.localPending { |
| 764 | m.localPending = m.syncLocal(ctx, m.snapshot()) |
| 765 | } |
| 766 | return nil |
| 767 | } |
| 768 | return m.refreshChanged(ctx, gen) |
| 769 | } |
| 770 | |
| 771 | // refreshChanged rebuilds the snapshot from one ETag listing plus a get per |
| 772 | // new or rewritten object, under refreshMu. gen is the control/certs |
| 773 | // generation captured before the listing. It keeps every guarantee of the |
| 774 | // full rebuild: |
| 775 | // |
| 776 | // - a key absent from the listing (a spent challenge token) is absent from |
| 777 | // the new snapshot, because the snapshot is built from the listing; |
| 778 | // - a failed get of a changed object carries the previous copy forward |
| 779 | // with an unknown ETag, so the next poll fetches it again, and is never |
| 780 | // a failed poll, even when it is the only object that changed; |
| 781 | // - the new snapshot is built whole and swapped in by publish, which also |
| 782 | // copies changed material into the local cache and wakes the wildcard |
| 783 | // manager, exactly as Refresh does. |
| 784 | // |
| 785 | // A store without the ETag listing, or a listing that fails, is "unknown": |
| 786 | // today's full Refresh runs instead. |
| 787 | func (m *certMirror) refreshChanged(ctx context.Context, gen uint64) error { |
| 788 | lister, ok := m.s.(release.ETagLister) |
| 789 | if !ok { |
| 790 | return m.refreshFull(ctx) |
| 791 | } |
| 792 | listed, err := lister.ListETags(ctx, certPrefix) |
| 793 | if err != nil { |
| 794 | return m.refreshFull(ctx) |
| 795 | } |
| 796 | prev := m.snapshot() |
| 797 | next := make(map[string][]byte, len(listed)) |
| 798 | etags := make(map[string]string, len(listed)) |
| 799 | var items []certFetch |
| 800 | for _, le := range listed { |
| 801 | name, ok := strings.CutPrefix(le.Key, certPrefix) |
| 802 | if !ok || !publishableCertName(name) { |
| 803 | continue |
| 804 | } |
| 805 | if was := m.etags[name]; was != "" && was == le.ETag { |
| 806 | if body, ok := prev.entries[name]; ok { |
| 807 | next[name], etags[name] = body, was |
| 808 | continue |
| 809 | } |
| 810 | } |
| 811 | items = append(items, certFetch{key: le.Key, name: name}) |
| 812 | } |
| 813 | tried := len(items) |
| 814 | failed, lastErr := applyCertFetches(items, m.fetchCerts(ctx, items), prev, next, etags) |
| 815 | // No "every object failed" rule here, unlike Refresh: the listing just |
| 816 | // answered, so the bucket is up, and a changed-only read is often ONE |
| 817 | // object. Treating that one failure as a bucket fault would turn a single |
| 818 | // unreadable key into a failed poll until the backstop, holding back |
| 819 | // every deletion in the listing and the loop's backfill with it. The |
| 820 | // failed objects carry their previous copies with unknown ETags and the |
| 821 | // generation stays unacted, so the next poll reads them again. |
| 822 | if failed > 0 { |
| 823 | m.logPartialPoll(failed, tried, lastErr) |
| 824 | } |
| 825 | m.publish(ctx, prev, next, etags) |
| 826 | m.actedGen, m.actedOK = gen, failed == 0 |
| 827 | return nil |
| 828 | } |
| 829 | |
| 830 | // certFetch is one object a refresh reads; certFetchResult its outcome. |
| 831 | type certFetch struct { |
| 832 | key, name string |
| 833 | } |
| 834 | |
| 835 | type certFetchResult struct { |
| 836 | body []byte |
| 837 | etag string |
| 838 | err error |
| 839 | } |
| 840 | |
| 841 | // fetchCerts gets every item's body, at most certRefreshConcurrency at once, |
| 842 | // and returns the results in item order. |
| 843 | func (m *certMirror) fetchCerts(ctx context.Context, items []certFetch) []certFetchResult { |
| 844 | results := make([]certFetchResult, len(items)) |
| 845 | // A fixed pool pulling indices off a channel, NOT a goroutine per object |
| 846 | // parked on a semaphore. The distinction only shows up at scale, and it is |
| 847 | // the whole point of having a bound: goroutine-per-object costs one stack |
| 848 | // per cert before a single one is read, so a deployment with a million |
| 849 | // published certs would spend gigabytes on goroutines that are all waiting |
| 850 | // for one of sixteen slots. |
| 851 | // |
| 852 | // Workers also select on ctx.Done, so a cancelled boot refresh stops |
| 853 | // handing out work instead of draining the whole queue through Get. (It |
| 854 | // still waits for the gets already in flight: Refresh has never been a |
| 855 | // hard bound on its own wall clock, and certMirrorBootTimeout bounds the |
| 856 | // requests, not this function's return.) |
| 857 | workers := min(certRefreshConcurrency, len(items)) |
| 858 | idx := make(chan int) |
| 859 | var wg sync.WaitGroup |
| 860 | for range workers { |
| 861 | wg.Add(1) |
| 862 | go func() { |
| 863 | defer wg.Done() |
| 864 | for i := range idx { |
| 865 | body, etag, err := m.s.Get(ctx, items[i].key) |
| 866 | results[i] = certFetchResult{body: body, etag: etag, err: err} |
| 867 | } |
| 868 | }() |
| 869 | } |
| 870 | for i := range items { |
| 871 | select { |
| 872 | case idx <- i: |
| 873 | case <-ctx.Done(): |
| 874 | // Leaves results[i:] zero — a nil body with a nil error, which |
| 875 | // the apply loop reads as a successful empty get. Set the |
| 876 | // context's error on the rest so they take the failed-get branch |
| 877 | // and carry the previous copy forward instead. |
| 878 | for j := i; j < len(items); j++ { |
| 879 | results[j] = certFetchResult{err: ctx.Err()} |
| 880 | } |
| 881 | close(idx) |
| 882 | wg.Wait() |
| 883 | return results |
| 884 | } |
| 885 | } |
| 886 | close(idx) |
| 887 | wg.Wait() |
| 888 | return results |
| 889 | } |
| 890 | |
| 891 | // applyCertFetches folds fetched bodies into next (and their ETags into |
| 892 | // etags), carrying prev's copy of any object that failed, and returns the |
| 893 | // failure count and the last failure. |
| 894 | func applyCertFetches(items []certFetch, results []certFetchResult, prev *certSnapshot, next map[string][]byte, etags map[string]string) (failed int, lastErr error) { |
| 895 | // Applied in key order, not completion order, so a partial poll reports |
| 896 | // the same lastErr whichever way the fetches happened to interleave. |
| 897 | for i, it := range items { |
| 898 | r := results[i] |
| 899 | switch { |
| 900 | case errors.Is(r.err, os.ErrNotExist): |
| 901 | // Raced a delete between the list and the get — a spent token. Not |
| 902 | // a fault; the entry is simply gone. |
| 903 | continue |
| 904 | case r.err != nil: |
| 905 | // ONE bad object is not a bad poll. Aborting here would discard the |
| 906 | // whole rebuild — every unrelated cert that landed this tick with |
| 907 | // it — and skip syncLocal and the wildcard wake; a single |
| 908 | // permanently unreadable key would freeze the mirror forever. Carry |
| 909 | // the previous copy of this name forward and keep going, with an |
| 910 | // unknown ETag so the next changed-only read fetches it again. |
| 911 | failed++ |
| 912 | lastErr = r.err |
| 913 | if body, ok := prev.entries[it.name]; ok { |
| 914 | next[it.name], etags[it.name] = body, "" |
| 915 | } |
| 916 | continue |
| 917 | } |
| 918 | next[it.name], etags[it.name] = r.body, r.etag |
| 919 | } |
| 920 | return failed, lastErr |
| 921 | } |
| 922 | |
| 923 | // publish swaps in the snapshot built from next, whole, and does what every |
| 924 | // landed refresh does after it: the local cache copy-in, the health latch and |
| 925 | // the wildcard wake. Under refreshMu. |
| 926 | func (m *certMirror) publish(ctx context.Context, prev *certSnapshot, next map[string][]byte, etags map[string]string) { |
| 927 | snap := buildCertSnapshot(prev, next) |
| 928 | m.mu.Lock() |
| 929 | m.snap = snap |
| 930 | m.mu.Unlock() |
| 931 | m.etags = etags |
| 932 | m.localPending = m.syncLocal(ctx, snap) |
| 933 | // The local cache the handshake reads now holds what the bucket does: |
| 934 | // the certificate condition is met for good (§6). A later failed |
| 935 | // refresh keeps serving this copy and leaves the latch set. |
| 936 | m.health.CertsLoaded() |
| 937 | if m.wake != nil && wildcardMaterialChanged(prev, snap) { |
| 938 | // A takeover (or a first wildcard for a new account) reaches the |
| 939 | // serving path in one poll instead of up to twelve hours. |
| 940 | m.wake() |
| 941 | } |
| 942 | } |
| 943 | |
| 944 | // pollWatch polls w under a bound of its own (timeout, or |
| 945 | // release.ChangeIndexPollTimeout when 0) inside ctx. The watch is shared with |
| 946 | // the agent's tick, whose hung index GET would otherwise hold this loop, and |
| 947 | // the health heartbeat it stamps, behind the watch's poll lock. |
| 948 | func pollWatch(ctx context.Context, w *release.ChangeWatch, timeout time.Duration) ([]string, error) { |
| 949 | if timeout <= 0 { |
| 950 | timeout = release.ChangeIndexPollTimeout |
| 951 | } |
| 952 | pctx, cancel := context.WithTimeout(ctx, timeout) |
| 953 | defer cancel() |
| 954 | return w.Poll(pctx) |
| 955 | } |
| 956 | |
| 957 | // certScope is the change-index scope every write under certPrefix bumps. |
| 958 | const certScope = "control/certs" |
| 959 | |
| 960 | func (m *certMirror) changeWatch() *release.ChangeWatch { |
| 961 | if m.watch == nil { |
| 962 | return nil |
| 963 | } |
| 964 | return m.watch() |
| 965 | } |
| 966 | |
| 967 | // certsGen is control/certs' generation as the watch last saw it, ok false |
| 968 | // when there is no usable watch. |
| 969 | func (m *certMirror) certsGen() (uint64, bool) { |
| 970 | if w := m.changeWatch(); w != nil { |
| 971 | return w.Gen(certScope) |
| 972 | } |
| 973 | return 0, false |
| 974 | } |
| 975 | |
| 976 | func (m *certMirror) clock() time.Time { |
| 977 | if m.now != nil { |
| 978 | return m.now() |
| 979 | } |
| 980 | return time.Now() |
| 981 | } |
| 982 | |
| 983 | // jitteredBackstop is release.ChangeIndexBackstop jittered by up to 10% either |
| 984 | // way (R3), drawn per full read so a fleet's rebuilds do not line up. |
| 985 | func jitteredBackstop() time.Duration { |
| 986 | b := release.ChangeIndexBackstop |
| 987 | return b - b/10 + time.Duration(rand.Int64N(int64(b/5)+1)) |
| 988 | } |
| 989 | |
| 990 | // logPartialPoll reports unreadable objects on the loop's own first-and-every- |
| 991 | // tenth cadence: a permanently bad key would otherwise say the same thing |
| 992 | // every 30 seconds forever. |
| 993 | func (m *certMirror) logPartialPoll(failed, tried int, err error) { |
| 994 | m.mu.Lock() |
| 995 | m.partialPolls++ |
| 996 | n := m.partialPolls |
| 997 | m.mu.Unlock() |
| 998 | if n == 1 || n%10 == 0 { |
| 999 | log.Printf("certsync: %d of %d object(s) under %s unreadable this poll (#%d, last: %v) — the previous copy of each is carried forward", failed, tried, certPrefix, n, err) |
| 1000 | } |
| 1001 | } |
| 1002 | |
| 1003 | // buildCertSnapshot parses what changed and carries the rest of the previous |
| 1004 | // parse forward — a handshake never pays for x509 parsing, and a quiet |
| 1005 | // deployment pays for it once. |
| 1006 | func buildCertSnapshot(prev *certSnapshot, entries map[string][]byte) *certSnapshot { |
| 1007 | snap := &certSnapshot{entries: entries, certs: make(map[string]*tls.Certificate, len(entries))} |
| 1008 | for name, body := range entries { |
| 1009 | if !certShapedName(name) { |
| 1010 | continue |
| 1011 | } |
| 1012 | if strings.HasSuffix(name, alpnTokenSuffix) { |
| 1013 | // tls-alpn-01 token certs are never served from a snapshot — see |
| 1014 | // alpnTokenCert — so parsing them here would be work done every |
| 1015 | // poll for an answer nothing may use. Leaving them unparsed also |
| 1016 | // makes serving a stale one structurally impossible. |
| 1017 | continue |
| 1018 | } |
| 1019 | if _, seen := prev.entries[name]; seen && bytes.Equal(prev.entries[name], body) { |
| 1020 | // Unchanged. Carry the previous RESULT forward, failure included: |
| 1021 | // re-parsing identical bytes cannot answer differently, and |
| 1022 | // re-logging a corrupt entry every 30s forever is not "loud", it is |
| 1023 | // noise. |
| 1024 | if old, ok := prev.certs[name]; ok { |
| 1025 | snap.certs[name] = old |
| 1026 | } |
| 1027 | continue |
| 1028 | } |
| 1029 | cert, err := parseCertEntry(body) |
| 1030 | if err != nil { |
| 1031 | // Loud, and skipped rather than served: a body that does not parse |
| 1032 | // is a body no handshake can use, and the next poll re-reads it. |
| 1033 | log.Printf("certsync: mirrored entry %q does not parse as a certificate (%v) — skipping it", name, err) |
| 1034 | continue |
| 1035 | } |
| 1036 | snap.certs[name] = cert |
| 1037 | } |
| 1038 | return snap |
| 1039 | } |
| 1040 | |
| 1041 | func (m *certMirror) snapshot() *certSnapshot { |
| 1042 | m.mu.RLock() |
| 1043 | defer m.mu.RUnlock() |
| 1044 | return m.snap |
| 1045 | } |
| 1046 | |
| 1047 | // Cert returns the mirrored body for a cache name. Account keys are refused |
| 1048 | // here, not filtered at poll time, because the local sync legitimately needs |
| 1049 | // them: the refusal belongs at the SERVING door. |
| 1050 | func (m *certMirror) Cert(name string) ([]byte, bool) { |
| 1051 | if accountKeyName(name) { |
| 1052 | return nil, false |
| 1053 | } |
| 1054 | b, ok := m.snapshot().entries[name] |
| 1055 | return b, ok |
| 1056 | } |
| 1057 | |
| 1058 | // Challenge returns the HTTP-01 response body for token, if the mirror holds |
| 1059 | // it. The cache name is autocert's own (autocert.go:896, |
| 1060 | // httpTokenCacheKey): path.Base(<challenge path>) + "+http-01" — flat, with |
| 1061 | // no /.well-known/ prefix. |
| 1062 | func (m *certMirror) Challenge(token string) ([]byte, bool) { |
| 1063 | b, ok := m.snapshot().entries[token+httpTokenSuffix] |
| 1064 | return b, ok |
| 1065 | } |
| 1066 | |
| 1067 | const ( |
| 1068 | httpTokenSuffix = "+http-01" // autocert's HTTP-01 response entries |
| 1069 | alpnTokenSuffix = "+token" // autocert's tls-alpn-01 challenge certs |
| 1070 | // autocertAccountKeyName is autocert's own ACME account key entry |
| 1071 | // (autocert.go's accountKey cache name). wildcardAccountKeyName |
| 1072 | // (wildcard.go) is the DNS-01 manager's. |
| 1073 | autocertAccountKeyName = "acme_account+key" |
| 1074 | ) |
| 1075 | |
| 1076 | // accountKeyName reports whether name is ACME ACCOUNT material — the issuer's |
| 1077 | // identity at the CA, never a serving artifact. |
| 1078 | func accountKeyName(name string) bool { |
| 1079 | return name == autocertAccountKeyName || name == wildcardAccountKeyName |
| 1080 | } |
| 1081 | |
| 1082 | // certShapedName reports whether an entry is expected to parse as a |
| 1083 | // certificate: everything except account keys and HTTP-01 response bodies. |
| 1084 | // That leaves per-host certs (<host>, <host>+rsa), wildcard PEMs |
| 1085 | // (wildcard-<parent>.pem) and tls-alpn-01 token certs (<host>+token) — the |
| 1086 | // last of which only the live read-through parses (buildCertSnapshot skips |
| 1087 | // them). |
| 1088 | func certShapedName(name string) bool { |
| 1089 | return !accountKeyName(name) && !strings.HasSuffix(name, httpTokenSuffix) |
| 1090 | } |
| 1091 | |
| 1092 | // parseCertEntry parses an autocert cache entry — the private key PEM |
| 1093 | // followed by the chain, which is also wildcard.go's encodeWildcardPEM shape — |
| 1094 | // into a usable tls.Certificate. Both halves come out of the one blob, and |
| 1095 | // X509KeyPair is what checks that the key actually matches the leaf. |
| 1096 | func parseCertEntry(raw []byte) (*tls.Certificate, error) { |
| 1097 | cert, err := tls.X509KeyPair(raw, raw) |
| 1098 | if err != nil { |
| 1099 | return nil, err |
| 1100 | } |
| 1101 | if cert.Leaf == nil { |
| 1102 | // X509KeyPair has populated Leaf since Go 1.23; belt and braces, since |
| 1103 | // every serving decision below reads it. |
| 1104 | leaf, err := x509.ParseCertificate(cert.Certificate[0]) |
| 1105 | if err != nil { |
| 1106 | return nil, err |
| 1107 | } |
| 1108 | cert.Leaf = leaf |
| 1109 | } |
| 1110 | return &cert, nil |
| 1111 | } |
| 1112 | |
| 1113 | // run polls until stop closes — the edge's standing background-loop shape |
| 1114 | // (Watch): fixed ticker, errors counted and logged at the first and then |
| 1115 | // every tenth, recovery logged once. |
| 1116 | func (m *certMirror) run(stop <-chan struct{}) { |
| 1117 | log.Printf("edge: cert mirror armed (%s poll of %s*) — this box serves every cert and challenge the issuer publishes", certMirrorInterval, certPrefix) |
| 1118 | tick := time.NewTicker(certMirrorInterval) |
| 1119 | defer tick.Stop() |
| 1120 | errs := 0 |
| 1121 | for { |
| 1122 | select { |
| 1123 | case <-stop: |
| 1124 | return |
| 1125 | case <-tick.C: |
| 1126 | } |
| 1127 | ctx, cancel := context.WithTimeout(context.Background(), certMirrorTimeout) |
| 1128 | err := m.poll(ctx) |
| 1129 | cancel() |
| 1130 | if err != nil { |
| 1131 | errs++ |
| 1132 | if errs == 1 || errs%10 == 0 { |
| 1133 | log.Printf("edge: cert mirror refresh failed (#%d): %v — serving the last good copy", errs, err) |
| 1134 | } |
| 1135 | continue |
| 1136 | } |
| 1137 | if errs > 0 { |
| 1138 | log.Printf("edge: cert mirror refresh recovered after %d failure(s)", errs) |
| 1139 | errs = 0 |
| 1140 | } |
| 1141 | ctx, cancel = context.WithTimeout(context.Background(), certMirrorTimeout) |
| 1142 | m.backfill(ctx) |
| 1143 | cancel() |
| 1144 | } |
| 1145 | } |
| 1146 | |
| 1147 | // backfill publishes what this box's cache holds and the bucket does not — |
| 1148 | // every box's half of "every edge serves every cert" for material minted |
| 1149 | // BEFORE publishing existed (#291), on whichever box's disk it sits (#318). |
| 1150 | // |
| 1151 | // publishingCache publishes on Put, which is to say on issuance and renewal. |
| 1152 | // A deployment that arms a second edge after a year of single-box service |
| 1153 | // therefore starts with a bucket holding nothing but what was minted since |
| 1154 | // the gate went in: the flagship roll found 4 objects against 50 live |
| 1155 | // certs, and every handshake on a new region edge failed until renewal — |
| 1156 | // 30 to 60 days out. Renewal is the wrong repair for that; this is a |
| 1157 | // one-pass copy, run after every successful poll, that converges in one |
| 1158 | // tick. Steady state on the issuer is one directory listing per poll plus |
| 1159 | // a stat per entry: an entry whose size and mtime have not moved since it |
| 1160 | // was last settled — in sync with the bucket, or found not to be this |
| 1161 | // deployment's — is not re-read (m.settled). |
| 1162 | // |
| 1163 | // Rules, each doing work: |
| 1164 | // |
| 1165 | // - Only on a box that contends in the election, NOT only on the issuer |
| 1166 | // (m.mayBackfill: agent-run AND contending in the issuer election). |
| 1167 | // The 2026-08-27 rung-1 rehearsal (#318) found the issuer-only rule |
| 1168 | // left every cert minted on a box that never held the lease since |
| 1169 | // #291 sitting on that box's disk alone — exactly the bytes a |
| 1170 | // failover loses with the box. Cert entries in control/certs/** are |
| 1171 | // therefore multi-writer now, kept convergent by the freshness rules |
| 1172 | // below (bytes-equal settles; an older leaf never displaces a newer |
| 1173 | // one), and the writer map's housekeeper-only line for this prefix is |
| 1174 | // amended to "any contending box, certs only". A concurrent writer |
| 1175 | // racing a stale view CAN transiently land an older leaf over a newer |
| 1176 | // one; the identity half of fileStamp is what makes that transient — |
| 1177 | // the box holding the newer leaf un-settles on the changed bucket sum |
| 1178 | // and republishes next pass. The serving-side "held |
| 1179 | // or ungated" read is still the wrong predicate: a hand-run `carlos |
| 1180 | // edge` with a bucket but no agent is ungated for serving and must |
| 1181 | // NOT dump its disk into the deployment's prefix — m.mayBackfill is |
| 1182 | // false exactly there. One consequence to hold in mind when a cert |
| 1183 | // PURGE ever lands: any box's mirrored on-disk copy now resurrects a |
| 1184 | // deleted bucket entry on its next pass, so a purge must tombstone or |
| 1185 | // it will not stick. |
| 1186 | // - Only a DirCache is walked. The mirror holds the cache as an |
| 1187 | // autocert.Cache, and that interface cannot enumerate; the on-disk |
| 1188 | // layout is the only backend that exists (certcache.go) and the one |
| 1189 | // whose entries ARE plain files named by cache key. |
| 1190 | // - Only names this deployment owns publish. The disk of a long-lived box |
| 1191 | // holds a cert for every route ever detached or renamed (autocert never |
| 1192 | // deletes), and DirCache.Put leaves "<name><random>" temp files behind |
| 1193 | // a kill mid-write — neither is the deployment's to replicate onto every |
| 1194 | // edge. A per-host entry (<host>, <host>+rsa) must pass m.allowed, the |
| 1195 | // same registry-backed exact check both read-throughs use; a wildcard |
| 1196 | // PEM must be for a parent in the wildcard manager's current set and |
| 1197 | // its leaf must cover it. Account keys are their own shape. |
| 1198 | // - Challenge tokens never publish: they are seconds long, and a copy of |
| 1199 | // a spent one is litter every edge would go on serving. |
| 1200 | // - Account keys publish only when the bucket has none, and only from |
| 1201 | // the box holding the STRICT issuer gate (m.issuing) — the one |
| 1202 | // backfill decision that stays single-writer. The deployment's ACME |
| 1203 | // identity is whichever key was published first (the mirror seeds it |
| 1204 | // onto every box, and a takeover issuer orders under it); a box's own |
| 1205 | // older key must never displace it, or the deployment would swap |
| 1206 | // accounts on every failover — and N agent boxes racing an EMPTY |
| 1207 | // bucket with their own keys would leave certs ordered under one |
| 1208 | // account stored beside another's key, which is why the widened cert |
| 1209 | // rule above does not extend here. A non-issuer's unseeded key is |
| 1210 | // deliberately NOT settled: the box may hold the lease on a later |
| 1211 | // pass, and a settled stamp would sit on exactly the pass that could |
| 1212 | // finally publish it. |
| 1213 | // - A cert-shaped entry publishes when the bucket lacks it, or holds an |
| 1214 | // older leaf (NotAfter earlier than this box's) — the renewed-before- |
| 1215 | // the-gate case. Anything that does not parse, or has already expired, |
| 1216 | // stays on disk: a stale rehearsal name is nothing to distribute, and |
| 1217 | // the expiry check is what keeps this from re-publishing it forever. |
| 1218 | // - Failures are per-entry, retried next poll, and logged on the file's |
| 1219 | // first-and-every-tenth cadence: a write-denied bucket must not say so |
| 1220 | // every 30 seconds forever. |
| 1221 | func (m *certMirror) backfill(ctx context.Context) { |
| 1222 | if m.mayBackfill == nil || !m.mayBackfill() || m.s == nil { |
| 1223 | return |
| 1224 | } |
| 1225 | dir, ok := m.local.(autocert.DirCache) |
| 1226 | if !ok { |
| 1227 | m.backfillOnce.Do(func() { |
| 1228 | log.Printf("certsync: backfill off — the local cert cache is not an on-disk cache and cannot be walked; only certs minted from now on reach %s*", certPrefix) |
| 1229 | }) |
| 1230 | return |
| 1231 | } |
| 1232 | entries, err := os.ReadDir(string(dir)) |
| 1233 | if err != nil { |
| 1234 | if !errors.Is(err, os.ErrNotExist) { |
| 1235 | m.logBackfillFailure(0, err) |
| 1236 | } |
| 1237 | return |
| 1238 | } |
| 1239 | snap := m.snapshot() |
| 1240 | now := time.Now() |
| 1241 | var published, failed int |
| 1242 | var lastErr error |
| 1243 | for _, ent := range entries { |
| 1244 | name := ent.Name() |
| 1245 | if !ent.Type().IsRegular() || !publishableCertName(name) { |
| 1246 | continue |
| 1247 | } |
| 1248 | if strings.HasSuffix(name, httpTokenSuffix) || strings.HasSuffix(name, alpnTokenSuffix) { |
| 1249 | continue |
| 1250 | } |
| 1251 | remote, inBucket := snap.entries[name] |
| 1252 | info, err := ent.Info() |
| 1253 | if err != nil { |
| 1254 | continue |
| 1255 | } |
| 1256 | stamp := fileStamp{size: info.Size(), mtime: info.ModTime().UnixNano(), bucketSum: bucketSum(remote, inBucket)} |
| 1257 | // Settled at this stamp on an earlier pass — in sync with the |
| 1258 | // bucket, or not this deployment's to publish — and nothing has |
| 1259 | // moved since. The bucket copy's IDENTITY is part of the stamp: an |
| 1260 | // entry that was in sync but has since left the bucket, or been |
| 1261 | // overwritten by another box, re-reads — that re-read is the #318 |
| 1262 | // repair path (see fileStamp). This is what keeps steady state at |
| 1263 | // a listing plus a stat per entry. |
| 1264 | if m.settledAt(name) == stamp { |
| 1265 | continue |
| 1266 | } |
| 1267 | body, err := os.ReadFile(filepath.Join(string(dir), name)) |
| 1268 | if err != nil { |
| 1269 | failed++ |
| 1270 | lastErr = err |
| 1271 | continue |
| 1272 | } |
| 1273 | if inBucket && bytes.Equal(remote, body) { |
| 1274 | // Also the backstop for names the snapshot never parses: the |
| 1275 | // freshness compare below cannot see those, and without this |
| 1276 | // they would republish every poll. |
| 1277 | m.markSettled(name, stamp) |
| 1278 | continue |
| 1279 | } |
| 1280 | if accountKeyName(name) { |
| 1281 | if inBucket { |
| 1282 | m.markSettled(name, stamp) |
| 1283 | continue |
| 1284 | } |
| 1285 | if m.issuing == nil || !m.issuing() { |
| 1286 | // Only the elected issuer seeds the deployment's ACME |
| 1287 | // identity (the rule above). Not settled, deliberately — |
| 1288 | // see the same rule for why. |
| 1289 | continue |
| 1290 | } |
| 1291 | } else { |
| 1292 | cert, err := parseCertEntry(body) |
| 1293 | if err != nil || !now.Before(cert.Leaf.NotAfter) || !m.ownsCertName(name, cert) { |
| 1294 | // Expiry is a function of time, but a settled entry can only |
| 1295 | // move from live to expired, never back; and ownership can |
| 1296 | // return (a route re-attached), which is a registry change — |
| 1297 | // so an entry settled as not-owned is re-examined only when |
| 1298 | // its stamp moves. A re-attached route mints afresh anyway. |
| 1299 | m.markSettled(name, stamp) |
| 1300 | continue |
| 1301 | } |
| 1302 | if inBucket { |
| 1303 | if have := snap.certs[name]; have != nil && !have.Leaf.NotAfter.Before(cert.Leaf.NotAfter) { |
| 1304 | m.markSettled(name, stamp) |
| 1305 | continue |
| 1306 | } |
| 1307 | } |
| 1308 | } |
| 1309 | if err := putCert(ctx, m.s, name, body); err != nil { |
| 1310 | failed++ |
| 1311 | lastErr = err |
| 1312 | continue |
| 1313 | } |
| 1314 | // Not settled: the bucket side of the stamp changes on the next |
| 1315 | // poll, and the bytes.Equal path settles it then. |
| 1316 | published++ |
| 1317 | } |
| 1318 | if published > 0 { |
| 1319 | log.Printf("certsync: backfilled %d cache entr%s from this box into %s* — every edge serves them on its next poll", published, plural(published, "y", "ies"), certPrefix) |
| 1320 | } |
| 1321 | if failed > 0 { |
| 1322 | m.logBackfillFailure(failed, lastErr) |
| 1323 | } |
| 1324 | } |
| 1325 | |
| 1326 | // ownsCertName reports whether a cert-shaped cache entry is one this |
| 1327 | // deployment should replicate: a per-host entry for a name the registry |
| 1328 | // still serves (m.allowed — nil admits nothing, the safe default for a |
| 1329 | // mirror built without an Edge), or a wildcard PEM for a parent in the |
| 1330 | // wildcard manager's CURRENT parent set (m.wildcardParents — nil, no |
| 1331 | // manager, admits none) whose leaf covers that parent. |
| 1332 | func (m *certMirror) ownsCertName(name string, cert *tls.Certificate) bool { |
| 1333 | if parent, ok := strings.CutPrefix(name, "wildcard-"); ok && strings.HasSuffix(parent, ".pem") { |
| 1334 | parent = strings.TrimSuffix(parent, ".pem") |
| 1335 | if parent == "" || m.wildcardParents == nil || !slices.Contains(m.wildcardParents(), parent) { |
| 1336 | return false |
| 1337 | } |
| 1338 | return slices.Contains(cert.Leaf.DNSNames, "*."+parent) |
| 1339 | } |
| 1340 | host := strings.TrimSuffix(name, "+rsa") |
| 1341 | if m.allowed == nil || !m.allowed(host) { |
| 1342 | return false |
| 1343 | } |
| 1344 | return cert.Leaf.VerifyHostname(host) == nil |
| 1345 | } |
| 1346 | |
| 1347 | // fileStamp is the state a backfill pass last settled an entry at: the |
| 1348 | // file's size+mtime, and the IDENTITY of the bucket's copy (a sha256 of |
| 1349 | // its bytes; zero = absent), not merely its presence. The local half is |
| 1350 | // not a hash: DirCache writes are atomic renames, so a changed body is a |
| 1351 | // changed stamp (nanosecond mtime, so a same-size rewrite within one |
| 1352 | // second on a coarse filesystem is the only alias, and autocert never |
| 1353 | // rewrites a name that fast). The bucket half MUST be identity: with |
| 1354 | // cert entries multi-writer (#318), another box racing a stale view can |
| 1355 | // land an older leaf over a newer one this box already settled against — |
| 1356 | // a presence bool would leave the settled box blind to the clobber |
| 1357 | // forever (its file unmoved, the name still "in bucket"), while the sum |
| 1358 | // changes with the clobber, un-settles exactly the box holding the newer |
| 1359 | // leaf, and its next pass republishes it. Convergence within a poll or |
| 1360 | // two, newest leaf wins. |
| 1361 | type fileStamp struct { |
| 1362 | size int64 |
| 1363 | mtime int64 |
| 1364 | bucketSum [sha256.Size]byte |
| 1365 | } |
| 1366 | |
| 1367 | // bucketSum is fileStamp's bucket-identity half. The zero value means |
| 1368 | // "absent", which cannot collide with a real entry: nothing under |
| 1369 | // control/certs/ is ever zero bytes, and sha256 of any body is nonzero. |
| 1370 | func bucketSum(body []byte, inBucket bool) [sha256.Size]byte { |
| 1371 | if !inBucket { |
| 1372 | return [sha256.Size]byte{} |
| 1373 | } |
| 1374 | return sha256.Sum256(body) |
| 1375 | } |
| 1376 | |
| 1377 | func (m *certMirror) settledAt(name string) fileStamp { |
| 1378 | m.mu.RLock() |
| 1379 | defer m.mu.RUnlock() |
| 1380 | return m.settled[name] |
| 1381 | } |
| 1382 | |
| 1383 | func (m *certMirror) markSettled(name string, st fileStamp) { |
| 1384 | m.mu.Lock() |
| 1385 | defer m.mu.Unlock() |
| 1386 | if m.settled == nil { |
| 1387 | m.settled = map[string]fileStamp{} |
| 1388 | } |
| 1389 | m.settled[name] = st |
| 1390 | } |
| 1391 | |
| 1392 | // logBackfillFailure is logPartialPoll's cadence for the publish side. |
| 1393 | func (m *certMirror) logBackfillFailure(failed int, err error) { |
| 1394 | m.mu.Lock() |
| 1395 | m.backfillFailures++ |
| 1396 | n := m.backfillFailures |
| 1397 | m.mu.Unlock() |
| 1398 | if n == 1 || n%10 == 0 { |
| 1399 | log.Printf("certsync: backfill: %d entr%s not published (#%d, last: %v) — retried on the next poll", failed, plural(failed, "y", "ies"), n, err) |
| 1400 | } |
| 1401 | } |
| 1402 | |
| 1403 | func plural(n int, one, many string) string { |
| 1404 | if n == 1 { |
| 1405 | return one |
| 1406 | } |
| 1407 | return many |
| 1408 | } |
| 1409 | |
| 1410 | // syncLocal copies mirrored material into this box's OWN autocert cache. It |
| 1411 | // is the quiet half of the mirror, and the one that matters when nothing is |
| 1412 | // looking: |
| 1413 | // |
| 1414 | // - autocert's renewal timers bypass HostPolicy and check the cache FIRST |
| 1415 | // (renewal.go's do()), so a stood-down box whose timers are still armed |
| 1416 | // finds a fresh cert here and never reaches the CA; |
| 1417 | // - a box that boots, or crashes and restarts, has no election result for |
| 1418 | // the first ~20 minutes; a warm local cache serves through that window |
| 1419 | // even before the mirror map is consulted; |
| 1420 | // - the wildcard manager loads parents from this same cache (loadCached), |
| 1421 | // which is how a wildcard the issuer minted reaches a non-issuer's |
| 1422 | // handshakes at all. |
| 1423 | // |
| 1424 | // Never written locally: challenge tokens of either kind. They are seconds |
| 1425 | // long, the issuer owns their deletion, and a copy on every box's disk is |
| 1426 | // litter nothing sweeps. |
| 1427 | // |
| 1428 | // Writes are best-effort and loud: a full disk must not stop the mirror from |
| 1429 | // SERVING what it already holds in memory. It reports whether any entry could |
| 1430 | // not be reconciled, so the armed poll can retry an unchanged snapshot. |
| 1431 | func (m *certMirror) syncLocal(ctx context.Context, snap *certSnapshot) (failedAny bool) { |
| 1432 | if m.local == nil { |
| 1433 | return false |
| 1434 | } |
| 1435 | m.syncMu.Lock() |
| 1436 | defer m.syncMu.Unlock() |
| 1437 | var failed int |
| 1438 | var lastErr error |
| 1439 | for name, body := range snap.entries { |
| 1440 | if strings.HasSuffix(name, httpTokenSuffix) || strings.HasSuffix(name, alpnTokenSuffix) { |
| 1441 | continue |
| 1442 | } |
| 1443 | // Unchanged since the last reconcile: the local copy was already |
| 1444 | // settled then, and nothing but autocert itself (which only ever writes |
| 1445 | // fresher material) can have moved it since. Steady state costs no |
| 1446 | // local I/O at all. |
| 1447 | if prev, ok := m.synced[name]; ok && bytes.Equal(prev, body) { |
| 1448 | continue |
| 1449 | } |
| 1450 | switch err := m.reconcileLocal(ctx, name, body, snap.certs[name]); { |
| 1451 | case err == nil: |
| 1452 | m.synced[name] = body |
| 1453 | case errors.Is(err, errUnparseableEntry): |
| 1454 | // Already reported by the snapshot build; nothing to say twice. |
| 1455 | default: |
| 1456 | failed++ |
| 1457 | lastErr = err |
| 1458 | } |
| 1459 | } |
| 1460 | if failed > 0 { |
| 1461 | // One line per poll, not one per name: a local cache that has stopped |
| 1462 | // accepting writes fails for EVERY name at once, and the fault is the |
| 1463 | // cache, not the certificate. |
| 1464 | log.Printf("certsync: %d mirrored entry/entries could not be reconciled into this box's local cert cache (last: %v) — they are still served from memory", failed, lastErr) |
| 1465 | } |
| 1466 | // Forget names the bucket no longer holds, so the map tracks the prefix |
| 1467 | // rather than growing with every token this box has ever seen. |
| 1468 | for name := range m.synced { |
| 1469 | if _, ok := snap.entries[name]; !ok { |
| 1470 | delete(m.synced, name) |
| 1471 | } |
| 1472 | } |
| 1473 | return failed > 0 |
| 1474 | } |
| 1475 | |
| 1476 | // errUnparseableEntry marks the one reconcile failure that is the ENTRY's |
| 1477 | // fault rather than the local cache's — already reported once by the snapshot |
| 1478 | // build, and not worth a second line. |
| 1479 | var errUnparseableEntry = errors.New("certsync: mirrored entry does not parse") |
| 1480 | |
| 1481 | // reconcileLocal writes one entry into the local cache when the local copy is |
| 1482 | // worse. A nil error means the local copy can be considered settled. |
| 1483 | func (m *certMirror) reconcileLocal(ctx context.Context, name string, body []byte, cert *tls.Certificate) error { |
| 1484 | local, err := m.local.Get(ctx, name) |
| 1485 | if err != nil && !errors.Is(err, autocert.ErrCacheMiss) { |
| 1486 | // An unreadable local cache is a real fault (permissions, disk). |
| 1487 | return fmt.Errorf("read %q: %w", name, err) |
| 1488 | } |
| 1489 | have := err == nil |
| 1490 | if accountKeyName(name) { |
| 1491 | // ACME ACCOUNT material. Seeded when absent, so that a box winning the |
| 1492 | // election later takes over the DEPLOYMENT's account — one account, one |
| 1493 | // rate-limit bucket, which is the envelope the issuance gate's |
| 1494 | // no-fencing decision is priced against. Never overwritten: a box that |
| 1495 | // already registered its own account has orders and authorizations tied |
| 1496 | // to that key, and replacing it under them buys nothing. |
| 1497 | if have { |
| 1498 | return nil |
| 1499 | } |
| 1500 | if err := m.local.Put(ctx, name, body); err != nil { |
| 1501 | return fmt.Errorf("seed the deployment's ACME account key %q: %w", name, err) |
| 1502 | } |
| 1503 | log.Printf("certsync: seeded the deployment's ACME account key (%s) into the local cache", name) |
| 1504 | return nil |
| 1505 | } |
| 1506 | if cert == nil { |
| 1507 | // Did not parse (already reported by buildCertSnapshot). Writing an |
| 1508 | // unparseable body over a working local cert would be a downgrade. |
| 1509 | return errUnparseableEntry |
| 1510 | } |
| 1511 | if have { |
| 1512 | if cur, err := parseCertEntry(local); err == nil && !cur.Leaf.NotAfter.Before(cert.Leaf.NotAfter) { |
| 1513 | // The local copy is at least as fresh — this box may BE the issuer, |
| 1514 | // one write ahead of what it has published. Leave it alone. |
| 1515 | return nil |
| 1516 | } |
| 1517 | } |
| 1518 | if err := m.local.Put(ctx, name, body); err != nil { |
| 1519 | return fmt.Errorf("write %q: %w", name, err) |
| 1520 | } |
| 1521 | return nil |
| 1522 | } |
| 1523 | |
| 1524 | // wildcardPEMName reports whether an entry is a wildcard cert PEM |
| 1525 | // (wildcard.go's wildcardPEMKey shape). |
| 1526 | func wildcardPEMName(name string) bool { |
| 1527 | return strings.HasPrefix(name, "wildcard-") && strings.HasSuffix(name, ".pem") |
| 1528 | } |
| 1529 | |
| 1530 | // wildcardMaterialChanged reports whether this poll brought a wildcard PEM |
| 1531 | // that is new or different. Only then is the wildcard manager nudged: its own |
| 1532 | // loop sleeps 12h between passes, so a takeover — or a first wildcard for a |
| 1533 | // new account, minted on the issuer — would otherwise wait that long to reach |
| 1534 | // this box's handshakes. A withdrawal is not a wake: there is nothing new to |
| 1535 | // load, and the manager keeps serving what it has until it expires. |
| 1536 | func wildcardMaterialChanged(prev, next *certSnapshot) bool { |
| 1537 | for name, body := range next.entries { |
| 1538 | if !wildcardPEMName(name) { |
| 1539 | continue |
| 1540 | } |
| 1541 | if !bytes.Equal(prev.entries[name], body) { |
| 1542 | return true |
| 1543 | } |
| 1544 | } |
| 1545 | return false |
| 1546 | } |
| 1547 | |
| 1548 | // ---- serving: the four-layer chain ------------------------------------------ |
| 1549 | // |
| 1550 | // wrapGetCertificate is the mirror's half of the handshake path. The FULL |
| 1551 | // chain a hello meets, outermost first (RunWithActivatorReady composes it): |
| 1552 | // |
| 1553 | // wildcard (wildcard.go) → mirror → local autocert → local cache, direct |
| 1554 | // |
| 1555 | // The order is forced by autocert's own shape, not by preference. autocert |
| 1556 | // runs HostPolicy — which carries the issuance gate — BEFORE its cache |
| 1557 | // lookup (autocert.go:292), so on a non-issuer autocert serves NOTHING: not |
| 1558 | // what the mirror handed it, not what it holds on disk itself. Putting the |
| 1559 | // mirror behind it would therefore serve nothing at all on every box but one. |
| 1560 | // |
| 1561 | // The last layer is the local cache read DIRECTLY, bypassing autocert, and it |
| 1562 | // is not redundant: it is what a box serves when the bucket is unreachable at |
| 1563 | // boot (empty mirror) but its own disk still holds yesterday's certs, and |
| 1564 | // what covers the ~20 minute window after a boot or a crash while the |
| 1565 | // housekeeper election is undecided. |
| 1566 | // |
| 1567 | // The short-circuit ahead of autocert applies while this box is GATED, and |
| 1568 | // only then. autocert arms its renewal timers LAZILY, out of GetCertificate |
| 1569 | // (m.cert -> startRenew, autocert.go:464) — a mirror that answered every |
| 1570 | // hello would leave an ISSUER with no renewal timers at all, and its certs |
| 1571 | // would expire ninety days later with nothing in the journal. An ungated box |
| 1572 | // loses nothing by going through autocert first: syncLocal has already put |
| 1573 | // every mirrored cert in the cache autocert reads, and the mirror stays |
| 1574 | // behind it as a fallback for anything autocert cannot serve. |
| 1575 | func (m *certMirror) wrapGetCertificate(inner func(*tls.ClientHelloInfo) (*tls.Certificate, error)) func(*tls.ClientHelloInfo) (*tls.Certificate, error) { |
| 1576 | return func(hello *tls.ClientHelloInfo) (*tls.Certificate, error) { |
| 1577 | // Converted ONCE, here, and used by every layer below: the mirror |
| 1578 | // lookup, the tls-alpn-01 lookup and the local cache read all have to |
| 1579 | // agree with the name autocert published under. |
| 1580 | name := serverName(hello.ServerName) |
| 1581 | if len(hello.SupportedProtos) == 1 && hello.SupportedProtos[0] == acme.ALPNProto { |
| 1582 | // A tls-alpn-01 validation. autocert answers its OWN order from |
| 1583 | // memory; anything else is another edge's order, and the CA reached |
| 1584 | // this box because every edge answers the same names. |
| 1585 | cert, err := inner(hello) |
| 1586 | if err == nil { |
| 1587 | return cert, nil |
| 1588 | } |
| 1589 | if c := m.alpnTokenCert(helloContext(hello), name); c != nil { |
| 1590 | return c, nil |
| 1591 | } |
| 1592 | return nil, err |
| 1593 | } |
| 1594 | wantRSA := !helloSupportsECDSA(hello) |
| 1595 | if m.isGated() { |
| 1596 | if c := m.certificate(name, wantRSA); c != nil { |
| 1597 | return c, nil |
| 1598 | } |
| 1599 | } |
| 1600 | cert, err := inner(hello) |
| 1601 | if err == nil { |
| 1602 | return cert, nil |
| 1603 | } |
| 1604 | // Ungated and autocert could not answer (an order that failed, a name |
| 1605 | // it will not order for): the mirror is still the better answer than a |
| 1606 | // dropped handshake. Gated, this repeat is a cheap map miss. |
| 1607 | if c := m.certificate(name, wantRSA); c != nil { |
| 1608 | return c, nil |
| 1609 | } |
| 1610 | if c := m.localCertificate(helloContext(hello), name, wantRSA); c != nil { |
| 1611 | return c, nil |
| 1612 | } |
| 1613 | return nil, err |
| 1614 | } |
| 1615 | } |
| 1616 | |
| 1617 | // serverName normalizes a client-supplied name to the cache name autocert |
| 1618 | // would have used: lowercased, no trailing dot, and punycode via |
| 1619 | // idna.Lookup.ToASCII — the same conversion autocert applies before its own |
| 1620 | // cache lookup (autocert.go:307), because some clients (curl among them) send |
| 1621 | // unicode SNI verbatim and a cert is always published under the A-label. |
| 1622 | // |
| 1623 | // A name IDNA refuses is kept in its lowercased form rather than dropped: |
| 1624 | // every layer below treats it as an ordinary miss, which is what it is, and |
| 1625 | // the refusal belongs to autocert (which rejects it outright) rather than to |
| 1626 | // the mirror. |
| 1627 | func serverName(sni string) string { |
| 1628 | name := strings.ToLower(strings.TrimSuffix(sni, ".")) |
| 1629 | if ascii, err := idna.Lookup.ToASCII(name); err == nil { |
| 1630 | return ascii |
| 1631 | } |
| 1632 | return name |
| 1633 | } |
| 1634 | |
| 1635 | // isGated reads the issuance gate, defaulting to "gated" (see the field). |
| 1636 | func (m *certMirror) isGated() bool { |
| 1637 | return m.gated == nil || m.gated() |
| 1638 | } |
| 1639 | |
| 1640 | // helloContext is the handshake's context, or a background one for the |
| 1641 | // hand-built hellos tests use. |
| 1642 | func helloContext(hello *tls.ClientHelloInfo) context.Context { |
| 1643 | if ctx := hello.Context(); ctx != nil { |
| 1644 | return ctx |
| 1645 | } |
| 1646 | return context.Background() |
| 1647 | } |
| 1648 | |
| 1649 | // certificate returns a mirrored cert for name, or nil. The two cache names |
| 1650 | // are autocert's own ("<host>" for ECDSA, "<host>+rsa" for legacy clients); |
| 1651 | // the preferred one is tried first and the other accepted rather than |
| 1652 | // dropping the handshake, since on a non-issuer no order can produce the |
| 1653 | // missing type. |
| 1654 | func (m *certMirror) certificate(name string, wantRSA bool) *tls.Certificate { |
| 1655 | if name == "" { |
| 1656 | return nil |
| 1657 | } |
| 1658 | snap := m.snapshot() |
| 1659 | for _, key := range certEntryNames(name, wantRSA) { |
| 1660 | if c := snap.certs[key]; usableFor(c, name) { |
| 1661 | return c |
| 1662 | } |
| 1663 | } |
| 1664 | return nil |
| 1665 | } |
| 1666 | |
| 1667 | // certEntryNames lists the cache names that can hold a cert for host, most |
| 1668 | // preferred first. |
| 1669 | func certEntryNames(host string, wantRSA bool) [2]string { |
| 1670 | if wantRSA { |
| 1671 | return [2]string{host + "+rsa", host} |
| 1672 | } |
| 1673 | return [2]string{host, host + "+rsa"} |
| 1674 | } |
| 1675 | |
| 1676 | // usableFor reports whether cert can be served for name right now. An expired |
| 1677 | // or wrong-named entry is treated as absent so the chain keeps falling |
| 1678 | // through — a dead cert fails the handshake just as surely as no cert, and |
| 1679 | // the next layer may have a live one. |
| 1680 | func usableFor(cert *tls.Certificate, name string) bool { |
| 1681 | return cert != nil && cert.Leaf != nil && |
| 1682 | time.Now().Before(cert.Leaf.NotAfter) && |
| 1683 | cert.Leaf.VerifyHostname(name) == nil |
| 1684 | } |
| 1685 | |
| 1686 | // helloSupportsECDSA is autocert's own supportsECDSA (autocert.go), ported |
| 1687 | // because it is unexported there and the mirror must look under the SAME |
| 1688 | // cache name autocert would have chosen for this client. |
| 1689 | func helloSupportsECDSA(hello *tls.ClientHelloInfo) bool { |
| 1690 | if hello.SignatureSchemes != nil { |
| 1691 | ecdsaOK := false |
| 1692 | schemeLoop: |
| 1693 | for _, scheme := range hello.SignatureSchemes { |
| 1694 | const tlsECDSAWithSHA1 tls.SignatureScheme = 0x0203 |
| 1695 | switch scheme { |
| 1696 | case tlsECDSAWithSHA1, tls.ECDSAWithP256AndSHA256, |
| 1697 | tls.ECDSAWithP384AndSHA384, tls.ECDSAWithP521AndSHA512: |
| 1698 | ecdsaOK = true |
| 1699 | break schemeLoop |
| 1700 | } |
| 1701 | } |
| 1702 | if !ecdsaOK { |
| 1703 | return false |
| 1704 | } |
| 1705 | } |
| 1706 | if hello.SupportedCurves != nil { |
| 1707 | ecdsaOK := false |
| 1708 | for _, curve := range hello.SupportedCurves { |
| 1709 | if curve == tls.CurveP256 { |
| 1710 | ecdsaOK = true |
| 1711 | break |
| 1712 | } |
| 1713 | } |
| 1714 | if !ecdsaOK { |
| 1715 | return false |
| 1716 | } |
| 1717 | } |
| 1718 | for _, suite := range hello.CipherSuites { |
| 1719 | switch suite { |
| 1720 | case tls.TLS_ECDHE_ECDSA_WITH_RC4_128_SHA, |
| 1721 | tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA, |
| 1722 | tls.TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA, |
| 1723 | tls.TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256, |
| 1724 | tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256, |
| 1725 | tls.TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384, |
| 1726 | tls.TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305: |
| 1727 | return true |
| 1728 | } |
| 1729 | } |
| 1730 | return false |
| 1731 | } |
| 1732 | |
| 1733 | // localCertTTL bounds how long a last-resort read is remembered. Short |
| 1734 | // enough that a renewal reaching the local cache is picked up promptly, long |
| 1735 | // enough that a flood of handshakes for one name is not a file read each. |
| 1736 | const localCertTTL = 30 * time.Second |
| 1737 | |
| 1738 | type localCertEntry struct { |
| 1739 | cert *tls.Certificate |
| 1740 | at time.Time |
| 1741 | } |
| 1742 | |
| 1743 | // localCertificate is the last layer: this box's own autocert cache, read |
| 1744 | // DIRECTLY, because a gated autocert.Manager refuses the name before it ever |
| 1745 | // looks there. Both outcomes are remembered briefly so a busy name costs one |
| 1746 | // file read per TTL rather than one per handshake. |
| 1747 | func (m *certMirror) localCertificate(ctx context.Context, name string, wantRSA bool) *tls.Certificate { |
| 1748 | if m.local == nil || name == "" { |
| 1749 | return nil |
| 1750 | } |
| 1751 | for _, key := range certEntryNames(name, wantRSA) { |
| 1752 | m.localMu.Lock() |
| 1753 | e, ok := m.localCerts[key] |
| 1754 | fresh := ok && time.Since(e.at) < localCertTTL |
| 1755 | m.localMu.Unlock() |
| 1756 | if fresh { |
| 1757 | if usableFor(e.cert, name) { |
| 1758 | return e.cert |
| 1759 | } |
| 1760 | continue |
| 1761 | } |
| 1762 | var cert *tls.Certificate |
| 1763 | if raw, err := m.local.Get(ctx, key); err == nil { |
| 1764 | if c, err := parseCertEntry(raw); err == nil { |
| 1765 | cert = c |
| 1766 | } |
| 1767 | } |
| 1768 | m.localMu.Lock() |
| 1769 | if len(m.localCerts) > readThroughCacheMax { |
| 1770 | // Crude but bounded: the map is a cache, and every entry is |
| 1771 | // re-readable from disk. |
| 1772 | clear(m.localCerts) |
| 1773 | } |
| 1774 | m.localCerts[key] = localCertEntry{cert: cert, at: time.Now()} |
| 1775 | m.localMu.Unlock() |
| 1776 | if usableFor(cert, name) { |
| 1777 | // Worth saying: the mirror had nothing for a name this box is |
| 1778 | // serving anyway. Once a minute for the whole box, not once per |
| 1779 | // name — every host on a box whose bucket is unreachable lands here |
| 1780 | // at once, and one line is enough to know it. |
| 1781 | m.localMu.Lock() |
| 1782 | quiet := time.Since(m.lastLocalLog) < time.Minute |
| 1783 | if !quiet { |
| 1784 | m.lastLocalLog = time.Now() |
| 1785 | } |
| 1786 | m.localMu.Unlock() |
| 1787 | if !quiet { |
| 1788 | log.Printf("edge: serving %s from this box's local cert cache — the mirror has no copy and issuance is gated here", name) |
| 1789 | } |
| 1790 | return cert |
| 1791 | } |
| 1792 | } |
| 1793 | return nil |
| 1794 | } |
| 1795 | |
| 1796 | // ---- the :80 challenge path ------------------------------------------------- |
| 1797 | |
| 1798 | const acmeChallengePrefix = "/.well-known/acme-challenge/" |
| 1799 | |
| 1800 | // challengeHandler answers HTTP-01 challenges from the mirror BEFORE |
| 1801 | // autocert's own handler sees them, and delegates everything else. The order |
| 1802 | // is forced, again, by autocert: HTTPHandler runs HostPolicy first |
| 1803 | // (autocert.go:405), so on a non-issuer it 403s a challenge whose answer is |
| 1804 | // sitting in the mirror. Behind this, autocert still answers its OWN live |
| 1805 | // orders from memory, which is what the issuer needs. |
| 1806 | func (m *certMirror) challengeHandler(next http.Handler) http.Handler { |
| 1807 | return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 1808 | if !strings.HasPrefix(r.URL.Path, acmeChallengePrefix) { |
| 1809 | next.ServeHTTP(w, r) |
| 1810 | return |
| 1811 | } |
| 1812 | // path.Base is the token autocert itself would derive |
| 1813 | // (httpTokenCacheKey), so both handlers agree on the name. |
| 1814 | if body, ok := m.challengeResponse(r.Context(), requestHost(r.Host), path.Base(r.URL.Path)); ok { |
| 1815 | w.Header().Set("Content-Type", "text/plain; charset=utf-8") |
| 1816 | _, _ = w.Write(body) |
| 1817 | return |
| 1818 | } |
| 1819 | next.ServeHTTP(w, r) |
| 1820 | }) |
| 1821 | } |
| 1822 | |
| 1823 | // challengeResponse answers from the polled map, then — and this is the part |
| 1824 | // the 30s poll cannot do — reads through to the bucket. A validation's whole |
| 1825 | // life is a few seconds: the issuer writes the token, sends Accept, and the |
| 1826 | // CA arrives at whichever edge DNS handed it. A poll that runs every 30s |
| 1827 | // misses that window every time. |
| 1828 | func (m *certMirror) challengeResponse(ctx context.Context, host, token string) ([]byte, bool) { |
| 1829 | if body, ok := m.Challenge(token); ok { |
| 1830 | // Already polled: a map hit costs nothing and the token is the CA's own |
| 1831 | // 128+ bits of entropy, so it needs no host bound of its own. |
| 1832 | return body, true |
| 1833 | } |
| 1834 | if !validChallengeToken(token) { |
| 1835 | return nil, false |
| 1836 | } |
| 1837 | // The LIVE lookup is bounded by the host under validation, exactly as |
| 1838 | // autocert bounds its own token lookup (HTTPHandler runs hostPolicy on |
| 1839 | // r.Host first, autocert.go:405). A real CA always sends the domain it is |
| 1840 | // validating, so the bound costs a genuine validation nothing — and |
| 1841 | // without it an unauthenticated :80 flood sustains a bucket GET per |
| 1842 | // distinct path, forever, from anywhere. The bound is the DEPLOYMENT's |
| 1843 | // exact names, not this box's (m.allowed, Edge.deploymentHost): DNS for a |
| 1844 | // home-boxed route can point at any edge, and the one it points at is the |
| 1845 | // one the CA arrives at (#277). |
| 1846 | if m.allowed != nil && !m.allowed(host) { |
| 1847 | return nil, false |
| 1848 | } |
| 1849 | body, _, ok := m.readThrough(ctx, token+httpTokenSuffix, &m.rtHTTP) |
| 1850 | return body, ok |
| 1851 | } |
| 1852 | |
| 1853 | // requestHost is the Host header without its port, normalized the same way |
| 1854 | // SNI is (serverName) so it meets Edge.Allowed's exact registry map in the |
| 1855 | // shape routes are stored in. |
| 1856 | func requestHost(hostport string) string { |
| 1857 | host := hostport |
| 1858 | if h, _, err := net.SplitHostPort(host); err == nil { |
| 1859 | host = h |
| 1860 | } |
| 1861 | return serverName(host) |
| 1862 | } |
| 1863 | |
| 1864 | // validChallengeToken bounds what an unauthenticated URL path may turn into a |
| 1865 | // bucket read: ACME tokens are base64url of at least 128 bits of entropy, so |
| 1866 | // nothing shorter or otherwise shaped can be one. It also keeps the key |
| 1867 | // inside the flat prefix (publishableCertName's concern, from the other |
| 1868 | // side). |
| 1869 | func validChallengeToken(token string) bool { |
| 1870 | if len(token) < 16 || len(token) > 128 { |
| 1871 | return false |
| 1872 | } |
| 1873 | for _, r := range token { |
| 1874 | switch { |
| 1875 | case r >= 'a' && r <= 'z', r >= 'A' && r <= 'Z', r >= '0' && r <= '9', r == '-', r == '_': |
| 1876 | default: |
| 1877 | return false |
| 1878 | } |
| 1879 | } |
| 1880 | return true |
| 1881 | } |
| 1882 | |
| 1883 | // alpnTokenCert answers a tls-alpn-01 validation for an order some OTHER edge |
| 1884 | // is running. autocert prefers tls-alpn-01 over http-01 and only falls back |
| 1885 | // to it after a whole failed order (autocert.go's AuthorizeOrderLoop), so |
| 1886 | // without this every issuance in an N-edge deployment burns a failed |
| 1887 | // validation with probability (N-1)/N before it ever tries the HTTP path. |
| 1888 | // |
| 1889 | // Bounded harder than the HTTP-01 path, because the name here is |
| 1890 | // attacker-chosen SNI rather than a high-entropy token: only names this |
| 1891 | // deployment serves — this box's own routes, or another box's exactly-named |
| 1892 | // ones (m.allowed, Edge.deploymentHost) — are ever looked up. A stranger's |
| 1893 | // invented label under a mirrored catch-all is not one of them. |
| 1894 | func (m *certMirror) alpnTokenCert(ctx context.Context, host string) *tls.Certificate { |
| 1895 | if host == "" { |
| 1896 | return nil |
| 1897 | } |
| 1898 | if m.allowed != nil && !m.allowed(host) { |
| 1899 | return nil |
| 1900 | } |
| 1901 | // Deliberately NOT from the poll's snapshot. Unlike an HTTP-01 entry — |
| 1902 | // whose name carries the order's unique token — a token CERT is keyed by |
| 1903 | // hostname alone, so the previous order's entry for the same host is a |
| 1904 | // live collision, not a stale copy of the same answer. Serving it hands |
| 1905 | // the CA the wrong key authorization and burns the order. The snapshot is |
| 1906 | // a poll behind at best, and far worse if a gated Delete left litter |
| 1907 | // behind, so this material is only ever read live. |
| 1908 | _, cert, _ := m.readThrough(ctx, host+alpnTokenSuffix, &m.rtALPN) |
| 1909 | if usableFor(cert, host) { |
| 1910 | return cert |
| 1911 | } |
| 1912 | return nil |
| 1913 | } |
| 1914 | |
| 1915 | // ---- read-through: the bounded live lookup ---------------------------------- |
| 1916 | |
| 1917 | const ( |
| 1918 | // readThroughTimeout budgets one live bucket read. It runs inside a |
| 1919 | // handshake or a :80 request, so it is short by construction. |
| 1920 | readThroughTimeout = 2 * time.Second |
| 1921 | // readThroughHitTTL is about one poll: after that the poll itself holds |
| 1922 | // the entry (or it has been withdrawn, which is also the answer). It |
| 1923 | // applies to names that are unique per order — HTTP-01 tokens. |
| 1924 | readThroughHitTTL = 30 * time.Second |
| 1925 | // readThroughMissTTL keeps a scanner walking the challenge path from |
| 1926 | // turning into a bucket GET per request. |
| 1927 | readThroughMissTTL = 5 * time.Second |
| 1928 | // readThroughCacheMax bounds the memo; over it, the map is dropped whole. |
| 1929 | readThroughCacheMax = 512 |
| 1930 | // readThroughBurst / readThroughRefill are one budget's rate: distinct |
| 1931 | // names defeat the negative cache, so the last bound is a rate. Under a |
| 1932 | // flood the read-through degrades to "off" — the 30s poll still covers |
| 1933 | // everything that is not a live validation. |
| 1934 | // |
| 1935 | // There is one budget PER LISTENER (rtHTTP, rtALPN), never one shared: |
| 1936 | // :80's challenge path is reachable by anyone, and a flood there must not |
| 1937 | // spend the budget 443's tls-alpn-01 rescue needs. They fail |
| 1938 | // independently or not at all. |
| 1939 | readThroughBurst = 20 |
| 1940 | readThroughRefill = 5 // per second |
| 1941 | ) |
| 1942 | |
| 1943 | // rtBudget is one token bucket. Guarded by certMirror.rtMu. |
| 1944 | type rtBudget struct { |
| 1945 | tokens float64 |
| 1946 | last time.Time |
| 1947 | } |
| 1948 | |
| 1949 | // spend takes one token, reporting whether there was one. Caller holds rtMu. |
| 1950 | func (b *rtBudget) spend(now time.Time) bool { |
| 1951 | if b.last.IsZero() { |
| 1952 | b.tokens = readThroughBurst |
| 1953 | } else { |
| 1954 | b.tokens = min(readThroughBurst, b.tokens+now.Sub(b.last).Seconds()*readThroughRefill) |
| 1955 | } |
| 1956 | b.last = now |
| 1957 | if b.tokens < 1 { |
| 1958 | return false |
| 1959 | } |
| 1960 | b.tokens-- |
| 1961 | return true |
| 1962 | } |
| 1963 | |
| 1964 | type readThroughEntry struct { |
| 1965 | body []byte |
| 1966 | cert *tls.Certificate |
| 1967 | ok bool |
| 1968 | at time.Time |
| 1969 | } |
| 1970 | |
| 1971 | // readThroughTokenTTL is how long a tls-alpn-01 token CERT may be remembered. |
| 1972 | // Seconds, not a poll: the name is the bare hostname, so the next order for |
| 1973 | // that host reuses it, and two orders can be seconds apart. A var so tests |
| 1974 | // can zero it. |
| 1975 | var readThroughTokenTTL = 3 * time.Second |
| 1976 | |
| 1977 | // readThroughTTL is how long a HIT for name may be reused. |
| 1978 | func readThroughTTL(name string) time.Duration { |
| 1979 | if strings.HasSuffix(name, alpnTokenSuffix) { |
| 1980 | return readThroughTokenTTL |
| 1981 | } |
| 1982 | return readThroughHitTTL |
| 1983 | } |
| 1984 | |
| 1985 | func (e readThroughEntry) fresh(name string) bool { |
| 1986 | ttl := readThroughMissTTL |
| 1987 | if e.ok { |
| 1988 | ttl = readThroughTTL(name) |
| 1989 | } |
| 1990 | return time.Since(e.at) < ttl |
| 1991 | } |
| 1992 | |
| 1993 | // readThrough fetches one control/certs/ entry outside the poll cycle: memoed |
| 1994 | // (hits and misses), single-flighted per name, rate-limited globally, and |
| 1995 | // budgeted per call. Errors are answers here, not faults — a bucket that is |
| 1996 | // down simply means this box cannot help with someone else's live |
| 1997 | // validation, and the CA retries. |
| 1998 | func (m *certMirror) readThrough(ctx context.Context, name string, budget *rtBudget) ([]byte, *tls.Certificate, bool) { |
| 1999 | m.rtMu.Lock() |
| 2000 | if e, ok := m.rt[name]; ok && e.fresh(name) { |
| 2001 | m.rtMu.Unlock() |
| 2002 | return e.body, e.cert, e.ok |
| 2003 | } |
| 2004 | if done, ok := m.rtInflight[name]; ok { |
| 2005 | // Somebody is already asking. The CA opens several connections for one |
| 2006 | // validation; they share the one bucket read. |
| 2007 | m.rtMu.Unlock() |
| 2008 | select { |
| 2009 | case <-done: |
| 2010 | case <-ctx.Done(): |
| 2011 | return nil, nil, false |
| 2012 | } |
| 2013 | m.rtMu.Lock() |
| 2014 | e := m.rt[name] |
| 2015 | m.rtMu.Unlock() |
| 2016 | return e.body, e.cert, e.ok |
| 2017 | } |
| 2018 | if !budget.spend(time.Now()) { |
| 2019 | m.rtMu.Unlock() |
| 2020 | return nil, nil, false |
| 2021 | } |
| 2022 | done := make(chan struct{}) |
| 2023 | m.rtInflight[name] = done |
| 2024 | m.rtMu.Unlock() |
| 2025 | |
| 2026 | // Detached from the caller: a client that hangs up mid-validation must not |
| 2027 | // cancel the read every other in-flight waiter is sharing. |
| 2028 | fetchCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), readThroughTimeout) |
| 2029 | body, _, err := m.s.Get(fetchCtx, certKey(name)) |
| 2030 | cancel() |
| 2031 | e := readThroughEntry{at: time.Now()} |
| 2032 | switch { |
| 2033 | case err == nil: |
| 2034 | e.body, e.ok = body, true |
| 2035 | if certShapedName(name) { |
| 2036 | if cert, perr := parseCertEntry(body); perr == nil { |
| 2037 | e.cert = cert |
| 2038 | } else { |
| 2039 | log.Printf("certsync: live read of %q does not parse as a certificate: %v", name, perr) |
| 2040 | } |
| 2041 | } |
| 2042 | case errors.Is(err, os.ErrNotExist): |
| 2043 | // The ordinary case: no such token. Negative-cached, not logged. |
| 2044 | default: |
| 2045 | m.logReadThroughError(name, err) |
| 2046 | } |
| 2047 | m.rtMu.Lock() |
| 2048 | if len(m.rt) > readThroughCacheMax { |
| 2049 | clear(m.rt) |
| 2050 | } |
| 2051 | m.rt[name] = e |
| 2052 | delete(m.rtInflight, name) |
| 2053 | m.rtMu.Unlock() |
| 2054 | close(done) |
| 2055 | return e.body, e.cert, e.ok |
| 2056 | } |
| 2057 | |
| 2058 | // logReadThroughError says once a minute that live lookups are failing. Not |
| 2059 | // per call: the rate limiter still allows several a second, and a bucket |
| 2060 | // outage is one fault however many validations hit it. |
| 2061 | func (m *certMirror) logReadThroughError(name string, err error) { |
| 2062 | m.rtMu.Lock() |
| 2063 | quiet := time.Since(m.lastRTLog) < time.Minute |
| 2064 | if !quiet { |
| 2065 | m.lastRTLog = time.Now() |
| 2066 | } |
| 2067 | m.rtMu.Unlock() |
| 2068 | if !quiet { |
| 2069 | log.Printf("certsync: live read of %q failed: %v — the %s poll still covers everything but live validations", name, err, certMirrorInterval) |
| 2070 | } |
| 2071 | } |
| 2072 | |
amadan