| 1 | // Cert sweep (#299 §3f, #243): the issuer removes cert-prefix entries that |
| 2 | // are provably dead — and nothing else. |
| 3 | // |
| 4 | // Two kinds of litter accumulate under control/certs/, for two different |
| 5 | // reasons, and they need two different predicates: |
| 6 | // |
| 7 | // - SPENT CHALLENGE TOKENS (#243). The issuer writes an HTTP-01 response |
| 8 | // or a tls-alpn-01 token cert, validates, and deletes it — but the |
| 9 | // delete is housekeeper-gated, so a stand-down or a crash mid-order |
| 10 | // leaves the token behind permanently. A validation's whole life is |
| 11 | // seconds, so AGE is a complete predicate here. |
| 12 | // - CERTS FOR HOSTS NOBODY SERVES ANY MORE. A detached custom domain's |
| 13 | // cert is never republished (ownsCertName refuses) and never deleted, |
| 14 | // so the object lingers, the mirror serves it to matching SNI until it |
| 15 | // expires, and the expired object then lingers forever. |
| 16 | // |
| 17 | // THE PREDICATE FOR THE SECOND IS EXPIRY, NEVER DETACHMENT, and the reason |
| 18 | // is worth stating at length because the tempting version of this sweep is |
| 19 | // catastrophic. A sweep keyed "no claim exists for this host" would delete |
| 20 | // every non-claim-backed cert in the deployment — every platform host, every |
| 21 | // wildcard PEM, the account keys — and worse, a registry lag or a stale |
| 22 | // edges read would take a LIVE host's certs down fleet-wide through the |
| 23 | // mirror. This repo's standing rule is that inference never destroys owner |
| 24 | // property (internal/domains/reconcile.go §1c), and a certificate is the |
| 25 | // member's property. |
| 26 | // |
| 27 | // So: an entry is deleted only when its own leaf expired long ago. That |
| 28 | // dissolves the "the mirror serves it until expiry" concern by |
| 29 | // construction, and it makes a wrong deletion harmless — the entry was |
| 30 | // already dead, and anything still wanted is re-minted by the warm. |
| 31 | // |
| 32 | // Account keys are never touched, by name and again by shape: they are the |
| 33 | // issuer's identity at the CA, and deleting one costs the deployment its |
| 34 | // ACME registration. |
| 35 | package edge |
| 36 | |
| 37 | import ( |
| 38 | "context" |
| 39 | "log" |
| 40 | "strings" |
| 41 | "time" |
| 42 | |
| 43 | "github.com/carlosframework/platform/internal/release" |
| 44 | ) |
| 45 | |
| 46 | const ( |
| 47 | // tokenLitterTTL is how long a spent challenge token may linger before |
| 48 | // the sweep takes it (#243). A validation completes in seconds and |
| 49 | // autocert's own order context is five minutes, so an hour is generous |
| 50 | // by two orders of magnitude — deliberately, because deleting a token |
| 51 | // out from under a live validation burns the order. |
| 52 | tokenLitterTTL = time.Hour |
| 53 | // certSweepParseAge is the pre-filter: an object younger than this is |
| 54 | // never even read, let alone deleted. |
| 55 | // |
| 56 | // It is what keeps the sweep from being the per-object N+1 it would |
| 57 | // otherwise be. autocert renews at ~30 days remaining on a 90-day |
| 58 | // certificate, so a HEALTHY entry is rewritten roughly every 60 days and |
| 59 | // its object stamp never gets this old. An entry that does is one |
| 60 | // nothing is renewing — which is exactly the population worth reading. |
| 61 | // Steady-state cost: one listing, zero reads. |
| 62 | certSweepParseAge = 60 * 24 * time.Hour |
| 63 | // certSweepGrace is how long a leaf must have been EXPIRED before its |
| 64 | // entry is removed. A month, because there is no hurry whatsoever: the |
| 65 | // cost of keeping a dead certificate is bytes, and the cost of removing |
| 66 | // a live one is an outage. |
| 67 | certSweepGrace = 30 * 24 * time.Hour |
| 68 | // certSweepInterval is how often the expiry half runs — the half that |
| 69 | // costs reads. A certificate dead for a month can wait a day. |
| 70 | certSweepInterval = 24 * time.Hour |
| 71 | // certSweepMinInterval is the floor for the whole sweep, token half |
| 72 | // included. It matches the warm's own quiet-path cadence, so an ordinary |
| 73 | // pass sweeps and a capped-drain pass (which comes back every minute |
| 74 | // until a backlog clears) does not. |
| 75 | certSweepMinInterval = warmInterval |
| 76 | // certSweepDeletesPerPass bounds the damage any single pass can do, in |
| 77 | // either direction: a bug here deletes at most this many objects before |
| 78 | // somebody sees the log, and a genuine backlog drains over days rather |
| 79 | // than in one burst against the bucket. |
| 80 | certSweepDeletesPerPass = 50 |
| 81 | ) |
| 82 | |
| 83 | // sweep removes provably dead entries under the cert prefix. Issuer-gated by |
| 84 | // its caller; see certWarm.pass. |
| 85 | // |
| 86 | // Never returns an error. Every failure is logged and skipped: a sweep is |
| 87 | // housekeeping, and housekeeping that can fail a pass would take the warm |
| 88 | // down with it. |
| 89 | func (c *certWarm) sweep(ctx context.Context, now time.Time) { |
| 90 | if c.store == nil { |
| 91 | return |
| 92 | } |
| 93 | // Not on a capped-drain pass. When a first-time backlog trips |
| 94 | // warmOrdersPerPass, pass() comes back every minute until it clears, and |
| 95 | // housekeeping does not need doing sixty times an hour — the token TTL |
| 96 | // is an hour and the expiry half is daily. Without this the drain would |
| 97 | // spend a listing of the whole cert prefix per minute for nothing. |
| 98 | if !c.lastSweep.IsZero() && now.Sub(c.lastSweep) < certSweepMinInterval { |
| 99 | return |
| 100 | } |
| 101 | lister, ok := c.store.(release.ModLister) |
| 102 | if !ok { |
| 103 | // No stamps, no age, no sweep. Guessing at an object's age from its |
| 104 | // contents is exactly the inference this file refuses to make. |
| 105 | return |
| 106 | } |
| 107 | ctx, cancel := context.WithTimeout(ctx, certSweepTimeout) |
| 108 | defer cancel() |
| 109 | entries, err := lister.ListMod(ctx, certPrefix) |
| 110 | if err != nil { |
| 111 | log.Printf("edge: cert sweep: could not list %s: %v", certPrefix, err) |
| 112 | return |
| 113 | } |
| 114 | c.lastSweep = now |
| 115 | expiryDue := now.Sub(c.lastExpirySweep) >= certSweepInterval |
| 116 | deletes := 0 |
| 117 | for _, e := range entries { |
| 118 | if deletes >= certSweepDeletesPerPass { |
| 119 | log.Printf("edge: cert sweep: stopping at %d deletions this pass — the rest go next pass", deletes) |
| 120 | break |
| 121 | } |
| 122 | name, ok := strings.CutPrefix(e.Key, certPrefix) |
| 123 | if !ok || name == "" { |
| 124 | continue |
| 125 | } |
| 126 | // Account material, first and unconditionally. Never by age, never |
| 127 | // by expiry, never at all. |
| 128 | if accountKeyName(name) { |
| 129 | continue |
| 130 | } |
| 131 | switch { |
| 132 | case isChallengeToken(name): |
| 133 | if now.Sub(e.Mod) < tokenLitterTTL { |
| 134 | continue |
| 135 | } |
| 136 | if c.deleteEntry(ctx, e.Key, "spent challenge token") { |
| 137 | deletes++ |
| 138 | } |
| 139 | case expiryDue && now.Sub(e.Mod) >= certSweepParseAge: |
| 140 | if c.sweepExpired(ctx, e.Key, name, now) { |
| 141 | deletes++ |
| 142 | } |
| 143 | } |
| 144 | } |
| 145 | if expiryDue { |
| 146 | c.lastExpirySweep = now |
| 147 | } |
| 148 | } |
| 149 | |
| 150 | // certSweepTimeout budgets one sweep. It runs on the warm's own goroutine, |
| 151 | // so a hung bucket must not hold the next warm pass hostage. |
| 152 | const certSweepTimeout = 2 * time.Minute |
| 153 | |
| 154 | // isChallengeToken reports whether name is validation scratch rather than a |
| 155 | // serving artifact — an HTTP-01 response body or a tls-alpn-01 token cert. |
| 156 | func isChallengeToken(name string) bool { |
| 157 | return strings.HasSuffix(name, httpTokenSuffix) || strings.HasSuffix(name, alpnTokenSuffix) |
| 158 | } |
| 159 | |
| 160 | // sweepExpired reads one long-untouched entry and removes it only if its own |
| 161 | // leaf says it is long dead. |
| 162 | // |
| 163 | // Everything that is not a confident yes is a no. An entry that will not |
| 164 | // read, will not parse, or carries no leaf is LEFT ALONE — it may be a shape |
| 165 | // a newer build publishes, and a sweep that deletes what it cannot |
| 166 | // understand is a sweep that deletes tomorrow's format. |
| 167 | func (c *certWarm) sweepExpired(ctx context.Context, key, name string, now time.Time) bool { |
| 168 | raw, _, err := c.store.Get(ctx, key) |
| 169 | if err != nil { |
| 170 | log.Printf("edge: cert sweep: could not read %s: %v", key, err) |
| 171 | return false |
| 172 | } |
| 173 | cert, err := parseCertEntry(raw) |
| 174 | if err != nil || cert.Leaf == nil { |
| 175 | log.Printf("edge: cert sweep: %s does not parse as a certificate, leaving it alone", key) |
| 176 | return false |
| 177 | } |
| 178 | if now.Sub(cert.Leaf.NotAfter) < certSweepGrace { |
| 179 | return false |
| 180 | } |
| 181 | return c.deleteEntry(ctx, key, "certificate expired "+cert.Leaf.NotAfter.UTC().Format("2006-01-02")+" for "+name) |
| 182 | } |
| 183 | |
| 184 | // deleteEntry removes one object and says so. Every deletion under this |
| 185 | // prefix is logged with its reason — there is no such thing as a quiet |
| 186 | // delete of cert material. |
| 187 | func (c *certWarm) deleteEntry(ctx context.Context, key, why string) bool { |
| 188 | if err := c.store.Delete(ctx, key); err != nil { |
| 189 | log.Printf("edge: cert sweep: could not delete %s: %v", key, err) |
| 190 | return false |
| 191 | } |
| 192 | log.Printf("edge: cert sweep: removed %s — %s", key, why) |
| 193 | return true |
| 194 | } |
| 195 | |
amadan