| 1 | package edge |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "os" |
| 7 | "testing" |
| 8 | "time" |
| 9 | |
| 10 | "github.com/carlosframework/platform/internal/release" |
| 11 | ) |
| 12 | |
| 13 | func sweepWarm(t *testing.T, s release.Store) *certWarm { |
| 14 | t.Helper() |
| 15 | return newCertWarm(func() []string { return nil }, nil, nil, nil, s) |
| 16 | } |
| 17 | |
| 18 | // seedCertEntry writes one cert-prefix object and backdates its stamp, which |
| 19 | // is how an object that has been sitting in the bucket for months looks. |
| 20 | func seedCertEntry(t *testing.T, dir string, s release.Store, name string, body []byte, age time.Duration) { |
| 21 | t.Helper() |
| 22 | if _, err := s.Put(context.Background(), certKey(name), body, ""); err != nil { |
| 23 | t.Fatal(err) |
| 24 | } |
| 25 | when := time.Now().Add(-age) |
| 26 | if err := os.Chtimes(dir+"/"+certKey(name), when, when); err != nil { |
| 27 | t.Fatal(err) |
| 28 | } |
| 29 | } |
| 30 | |
| 31 | func certGone(t *testing.T, s release.Store, name string) bool { |
| 32 | t.Helper() |
| 33 | _, _, err := s.Get(context.Background(), certKey(name)) |
| 34 | return errors.Is(err, os.ErrNotExist) |
| 35 | } |
| 36 | |
| 37 | func expiredPEM(t *testing.T, host string, expiredAgo time.Duration) []byte { |
| 38 | t.Helper() |
| 39 | cert, key := selfSignedLeaf(t, host, time.Now().Add(-expiredAgo)) |
| 40 | pem, err := encodeWildcardPEM(cert, key) |
| 41 | if err != nil { |
| 42 | t.Fatal(err) |
| 43 | } |
| 44 | return pem |
| 45 | } |
| 46 | |
| 47 | // #243: a spent token whose gated delete never ran. Age is a complete |
| 48 | // predicate — a validation's whole life is seconds. |
| 49 | func TestSweepTakesSpentChallengeTokens(t *testing.T) { |
| 50 | dir := t.TempDir() |
| 51 | s := release.NewDirStore(dir) |
| 52 | w := sweepWarm(t, s) |
| 53 | seedCertEntry(t, dir, s, "abc123"+httpTokenSuffix, []byte("key-authorization"), 2*time.Hour) |
| 54 | seedCertEntry(t, dir, s, "www.customer.test"+alpnTokenSuffix, []byte("whatever"), 2*time.Hour) |
| 55 | // A token from a validation that may still be running must survive. |
| 56 | seedCertEntry(t, dir, s, "fresh"+httpTokenSuffix, []byte("key-authorization"), time.Minute) |
| 57 | |
| 58 | w.sweep(context.Background(), time.Now()) |
| 59 | |
| 60 | if !certGone(t, s, "abc123"+httpTokenSuffix) { |
| 61 | t.Error("a spent HTTP-01 token survived the sweep") |
| 62 | } |
| 63 | if !certGone(t, s, "www.customer.test"+alpnTokenSuffix) { |
| 64 | t.Error("a spent tls-alpn-01 token cert survived the sweep") |
| 65 | } |
| 66 | if certGone(t, s, "fresh"+httpTokenSuffix) { |
| 67 | t.Error("a token from a possibly-live validation was deleted — that burns the order") |
| 68 | } |
| 69 | } |
| 70 | |
| 71 | // §3f's whole ruling. The predicate is the leaf's own expiry and nothing |
| 72 | // else: a cert for a host nobody serves any more goes ONLY once it is long |
| 73 | // dead, and a current cert is untouchable however old its object is. |
| 74 | func TestSweepDeletesOnlyLongExpiredCertificates(t *testing.T) { |
| 75 | dir := t.TempDir() |
| 76 | s := release.NewDirStore(dir) |
| 77 | w := sweepWarm(t, s) |
| 78 | old := certSweepParseAge + 24*time.Hour |
| 79 | |
| 80 | seedCertEntry(t, dir, s, "dead.customer.test", expiredPEM(t, "dead.customer.test", certSweepGrace+24*time.Hour), old) |
| 81 | // Expired, but not long enough ago. Patience costs bytes; haste costs |
| 82 | // an outage. |
| 83 | seedCertEntry(t, dir, s, "recent.customer.test", expiredPEM(t, "recent.customer.test", time.Hour), old) |
| 84 | // A CURRENT cert whose object happens to be ancient — a CA with long |
| 85 | // certs, or a renewal that has simply not been needed. |
| 86 | cert, key := selfSignedLeaf(t, "live.customer.test", time.Now().Add(365*24*time.Hour)) |
| 87 | livePEM, err := encodeWildcardPEM(cert, key) |
| 88 | if err != nil { |
| 89 | t.Fatal(err) |
| 90 | } |
| 91 | seedCertEntry(t, dir, s, "live.customer.test", livePEM, old) |
| 92 | |
| 93 | w.sweep(context.Background(), time.Now()) |
| 94 | |
| 95 | if !certGone(t, s, "dead.customer.test") { |
| 96 | t.Error("a certificate that expired a month ago survived") |
| 97 | } |
| 98 | if certGone(t, s, "recent.customer.test") { |
| 99 | t.Error("a certificate that expired an hour ago was deleted — the grace period is not being applied") |
| 100 | } |
| 101 | if certGone(t, s, "live.customer.test") { |
| 102 | t.Fatal("A CURRENT CERTIFICATE WAS DELETED. The predicate is expiry, never object age.") |
| 103 | } |
| 104 | } |
| 105 | |
| 106 | // The account keys are the issuer's identity at the CA. They are written |
| 107 | // once and never rewritten, so they are ALWAYS older than the parse floor — |
| 108 | // which is exactly why they are excluded by name before anything else is |
| 109 | // considered. Deleting one costs the deployment its ACME registration. |
| 110 | func TestSweepNeverTouchesAccountKeys(t *testing.T) { |
| 111 | dir := t.TempDir() |
| 112 | s := release.NewDirStore(dir) |
| 113 | w := sweepWarm(t, s) |
| 114 | ancient := 5 * 365 * 24 * time.Hour |
| 115 | seedCertEntry(t, dir, s, autocertAccountKeyName, []byte("account key material"), ancient) |
| 116 | seedCertEntry(t, dir, s, wildcardAccountKeyName, []byte("account key material"), ancient) |
| 117 | |
| 118 | w.sweep(context.Background(), time.Now()) |
| 119 | |
| 120 | for _, name := range []string{autocertAccountKeyName, wildcardAccountKeyName} { |
| 121 | if certGone(t, s, name) { |
| 122 | t.Fatalf("ACCOUNT KEY %s WAS DELETED — the deployment has lost its ACME registration", name) |
| 123 | } |
| 124 | } |
| 125 | } |
| 126 | |
| 127 | // A sweep that deletes what it cannot understand is a sweep that deletes |
| 128 | // tomorrow's format. Anything that will not read, will not parse, or has no |
| 129 | // leaf is left exactly where it is. |
| 130 | func TestSweepLeavesWhatItCannotUnderstand(t *testing.T) { |
| 131 | dir := t.TempDir() |
| 132 | s := release.NewDirStore(dir) |
| 133 | w := sweepWarm(t, s) |
| 134 | old := certSweepParseAge + 24*time.Hour |
| 135 | seedCertEntry(t, dir, s, "future-shape.customer.test", []byte("a format this build has never seen"), old) |
| 136 | |
| 137 | w.sweep(context.Background(), time.Now()) |
| 138 | |
| 139 | if certGone(t, s, "future-shape.customer.test") { |
| 140 | t.Error("an entry this build could not parse was deleted") |
| 141 | } |
| 142 | } |
| 143 | |
| 144 | // The pre-filter is what keeps this from being an N+1: nothing younger than |
| 145 | // the parse floor is even read. A healthy deployment renews every ~60 days, |
| 146 | // so the steady-state cost is one listing and zero reads. |
| 147 | func TestSweepReadsNothingYoungerThanTheParseFloor(t *testing.T) { |
| 148 | dir := t.TempDir() |
| 149 | s := &bucketGetCounter{Store: release.NewDirStore(dir)} |
| 150 | w := sweepWarm(t, s) |
| 151 | for _, host := range []string{"a.customer.test", "b.customer.test", "c.customer.test"} { |
| 152 | seedCertEntry(t, dir, s, host, expiredPEM(t, host, time.Hour), 10*24*time.Hour) |
| 153 | } |
| 154 | before := s.gets() |
| 155 | w.sweep(context.Background(), time.Now()) |
| 156 | if n := s.gets() - before; n != 0 { |
| 157 | t.Errorf("sweep spent %d get(s) on entries younger than the parse floor, want 0", n) |
| 158 | } |
| 159 | } |
| 160 | |
| 161 | // The expiry half runs daily, not hourly: it is the half that costs reads, |
| 162 | // and a certificate that has been dead for a month can wait a day. The token |
| 163 | // half runs every pass, because it needs no reads at all. |
| 164 | func TestSweepRunsTheExpiryHalfDailyAndTheTokenHalfEveryPass(t *testing.T) { |
| 165 | dir := t.TempDir() |
| 166 | s := release.NewDirStore(dir) |
| 167 | w := sweepWarm(t, s) |
| 168 | old := certSweepParseAge + 24*time.Hour |
| 169 | now := time.Now() |
| 170 | |
| 171 | seedCertEntry(t, dir, s, "dead-one.customer.test", expiredPEM(t, "dead-one.customer.test", certSweepGrace+24*time.Hour), old) |
| 172 | w.sweep(context.Background(), now) |
| 173 | if !certGone(t, s, "dead-one.customer.test") { |
| 174 | t.Fatal("the first sweep did not run the expiry half") |
| 175 | } |
| 176 | |
| 177 | // An hour later, a second dead cert appears and a token goes stale. |
| 178 | seedCertEntry(t, dir, s, "dead-two.customer.test", expiredPEM(t, "dead-two.customer.test", certSweepGrace+24*time.Hour), old) |
| 179 | seedCertEntry(t, dir, s, "tok"+httpTokenSuffix, []byte("key-authorization"), 2*time.Hour) |
| 180 | w.sweep(context.Background(), now.Add(time.Hour)) |
| 181 | if certGone(t, s, "dead-two.customer.test") { |
| 182 | t.Error("the expiry half ran again an hour later — it is meant to be daily") |
| 183 | } |
| 184 | if !certGone(t, s, "tok"+httpTokenSuffix) { |
| 185 | t.Error("the token half did not run on the second pass — it is meant to run every pass") |
| 186 | } |
| 187 | |
| 188 | // A day later it comes back round. |
| 189 | w.sweep(context.Background(), now.Add(certSweepInterval+time.Minute)) |
| 190 | if !certGone(t, s, "dead-two.customer.test") { |
| 191 | t.Error("the expiry half did not run again after a day") |
| 192 | } |
| 193 | } |
| 194 | |
| 195 | // A store with no last-modified stamps gives the sweep no age to judge by, |
| 196 | // and guessing an object's age from its contents is exactly the inference |
| 197 | // this file refuses to make. It does nothing at all rather than something |
| 198 | // approximate. |
| 199 | func TestSweepStandsDownWithoutStamps(t *testing.T) { |
| 200 | dir := t.TempDir() |
| 201 | inner := release.NewDirStore(dir) |
| 202 | w := sweepWarm(t, stamplessEdgeStore{inner}) |
| 203 | seedCertEntry(t, dir, inner, "tok"+httpTokenSuffix, []byte("key-authorization"), 30*24*time.Hour) |
| 204 | |
| 205 | w.sweep(context.Background(), time.Now()) |
| 206 | |
| 207 | if certGone(t, inner, "tok"+httpTokenSuffix) { |
| 208 | t.Error("a store with no stamps still had entries deleted") |
| 209 | } |
| 210 | } |
| 211 | |
| 212 | type stamplessEdgeStore struct{ inner release.Store } |
| 213 | |
| 214 | func (s stamplessEdgeStore) Get(ctx context.Context, key string) ([]byte, string, error) { |
| 215 | return s.inner.Get(ctx, key) |
| 216 | } |
| 217 | func (s stamplessEdgeStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| 218 | return s.inner.Put(ctx, key, body, ifMatch) |
| 219 | } |
| 220 | func (s stamplessEdgeStore) List(ctx context.Context, prefix string) ([]string, error) { |
| 221 | return s.inner.List(ctx, prefix) |
| 222 | } |
| 223 | func (s stamplessEdgeStore) ListDirs(ctx context.Context, prefix string) ([]string, error) { |
| 224 | return s.inner.ListDirs(ctx, prefix) |
| 225 | } |
| 226 | func (s stamplessEdgeStore) Delete(ctx context.Context, key string) error { |
| 227 | return s.inner.Delete(ctx, key) |
| 228 | } |
| 229 | |
| 230 | // A bug in the predicate deletes at most this many objects before somebody |
| 231 | // reads the log, and a genuine backlog drains over days rather than in one |
| 232 | // burst against the bucket. |
| 233 | func TestSweepBoundsHowMuchOnePassCanDelete(t *testing.T) { |
| 234 | dir := t.TempDir() |
| 235 | s := release.NewDirStore(dir) |
| 236 | w := sweepWarm(t, s) |
| 237 | total := certSweepDeletesPerPass + 5 |
| 238 | for i := 0; i < total; i++ { |
| 239 | seedCertEntry(t, dir, s, string(rune('a'+i%26))+string(rune('a'+i/26))+".tok"+httpTokenSuffix, |
| 240 | []byte("key-authorization"), 2*time.Hour) |
| 241 | } |
| 242 | w.sweep(context.Background(), time.Now()) |
| 243 | left, err := s.List(context.Background(), certPrefix) |
| 244 | if err != nil { |
| 245 | t.Fatal(err) |
| 246 | } |
| 247 | if len(left) != total-certSweepDeletesPerPass { |
| 248 | t.Errorf("%d entries left after one pass, want %d — the per-pass bound is not holding", |
| 249 | len(left), total-certSweepDeletesPerPass) |
| 250 | } |
| 251 | } |
| 252 | |
| 253 | // The sweep's work exists whether or not this deployment has custom |
| 254 | // domains: spent challenge tokens and long-expired certificates are not the |
| 255 | // warm SET's business. A pass over an empty set still sweeps — the gate check |
| 256 | // deliberately comes before the empty-set return in pass(). |
| 257 | func TestPassWithNoWarmSetStillSweeps(t *testing.T) { |
| 258 | dir := t.TempDir() |
| 259 | s := release.NewDirStore(dir) |
| 260 | w := sweepWarm(t, s) |
| 261 | seedCertEntry(t, dir, s, "orphan"+httpTokenSuffix, []byte("key-authorization"), 2*time.Hour) |
| 262 | |
| 263 | if d := w.pass(); d != warmInterval { |
| 264 | t.Fatalf("pass over an empty set returned %s", d) |
| 265 | } |
| 266 | if !certGone(t, s, "orphan"+httpTokenSuffix) { |
| 267 | t.Error("a deployment with no custom domains never sweeps its own token litter") |
| 268 | } |
| 269 | } |
| 270 | |
| 271 | // A non-issuer sweeps nothing. Deleting cert material is the sharpest thing |
| 272 | // this package does, and a box that is not the issuer has no authority to |
| 273 | // do it. |
| 274 | func TestSweepNeverRunsOnANonIssuer(t *testing.T) { |
| 275 | dir := t.TempDir() |
| 276 | s := release.NewDirStore(dir) |
| 277 | w := newCertWarm(func() []string { return nil }, nil, nil, |
| 278 | func() (context.Context, bool) { return nil, false }, s) |
| 279 | seedCertEntry(t, dir, s, "orphan"+httpTokenSuffix, []byte("key-authorization"), 2*time.Hour) |
| 280 | |
| 281 | w.pass() |
| 282 | |
| 283 | if certGone(t, s, "orphan"+httpTokenSuffix) { |
| 284 | t.Error("a stood-down box deleted cert material") |
| 285 | } |
| 286 | } |
| 287 | |
| 288 | // A first-time backlog trips warmOrdersPerPass and pass() comes back every |
| 289 | // minute until it clears. Housekeeping must not ride along sixty times an |
| 290 | // hour: the token TTL is an hour and the expiry half is daily, so a listing |
| 291 | // of the whole cert prefix per minute would be spent on nothing. |
| 292 | func TestSweepDoesNotRunOnEveryCappedDrainPass(t *testing.T) { |
| 293 | dir := t.TempDir() |
| 294 | s := &listCounter{Store: release.NewDirStore(dir)} |
| 295 | w := sweepWarm(t, s) |
| 296 | now := time.Now() |
| 297 | |
| 298 | w.sweep(context.Background(), now) |
| 299 | if s.lists != 1 { |
| 300 | t.Fatalf("first sweep listed %d time(s), want 1", s.lists) |
| 301 | } |
| 302 | w.sweep(context.Background(), now.Add(time.Minute)) |
| 303 | w.sweep(context.Background(), now.Add(2*time.Minute)) |
| 304 | if s.lists != 1 { |
| 305 | t.Errorf("sweep listed %d times across three passes a minute apart, want 1", s.lists) |
| 306 | } |
| 307 | w.sweep(context.Background(), now.Add(certSweepMinInterval+time.Second)) |
| 308 | if s.lists != 2 { |
| 309 | t.Errorf("sweep listed %d times, want 2 — an ordinary pass an hour later must sweep", s.lists) |
| 310 | } |
| 311 | } |
| 312 | |
| 313 | type listCounter struct { |
| 314 | release.Store |
| 315 | lists int |
| 316 | } |
| 317 | |
| 318 | func (l *listCounter) ListMod(ctx context.Context, prefix string) ([]release.KeyMod, error) { |
| 319 | l.lists++ |
| 320 | return l.Store.(release.ModLister).ListMod(ctx, prefix) |
| 321 | } |
| 322 | |
amadan