| 1 | package edge |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "errors" |
| 7 | "io" |
| 8 | "log" |
| 9 | "os" |
| 10 | "path/filepath" |
| 11 | "slices" |
| 12 | "strings" |
| 13 | "sync" |
| 14 | "testing" |
| 15 | "time" |
| 16 | |
| 17 | "golang.org/x/crypto/acme/autocert" |
| 18 | |
| 19 | "github.com/carlosframework/platform/internal/release" |
| 20 | ) |
| 21 | |
| 22 | // issuerMirror is a mirror over local+bucket whose box holds the issuer gate |
| 23 | // when held is true. Refresh has run, so the snapshot reflects the bucket. |
| 24 | func issuerMirror(t *testing.T, s release.Store, local autocert.Cache, held bool) *certMirror { |
| 25 | t.Helper() |
| 26 | m := newCertMirror(s, local) |
| 27 | m.mayBackfill = func() bool { return true } // an agent-run box |
| 28 | m.issuing = func() bool { return held } |
| 29 | // The registry: every *.example.com name except "gone". |
| 30 | m.allowed = func(host string) bool { |
| 31 | return strings.HasSuffix(host, ".example.com") && !strings.HasPrefix(host, "gone.") |
| 32 | } |
| 33 | // The wildcard manager's parent set: acct only. |
| 34 | m.wildcardParents = func() []string { return []string{"acct.example.com"} } |
| 35 | if err := m.Refresh(context.Background()); err != nil { |
| 36 | t.Fatal(err) |
| 37 | } |
| 38 | return m |
| 39 | } |
| 40 | |
| 41 | func putLocal(t *testing.T, c autocert.Cache, name string, body []byte) { |
| 42 | t.Helper() |
| 43 | if err := c.Put(context.Background(), name, body); err != nil { |
| 44 | t.Fatal(err) |
| 45 | } |
| 46 | } |
| 47 | |
| 48 | func bucketHas(t *testing.T, s release.Store, name string) ([]byte, bool) { |
| 49 | t.Helper() |
| 50 | b, err := certStoreGet(t, s, certKey(name)) |
| 51 | if err != nil { |
| 52 | return nil, false |
| 53 | } |
| 54 | return b, true |
| 55 | } |
| 56 | |
| 57 | // The flagship roll's shape: a cache full of certs minted before publishing |
| 58 | // existed, a bucket holding almost nothing. One pass and the bucket has |
| 59 | // every live cert — and nothing that should not be there. |
| 60 | func TestBackfillPublishesPreExistingCertsOnce(t *testing.T) { |
| 61 | log.SetOutput(io.Discard) |
| 62 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 63 | s := release.NewDirStore(t.TempDir()) |
| 64 | local := autocert.DirCache(t.TempDir()) |
| 65 | later := time.Now().Add(60 * 24 * time.Hour) |
| 66 | |
| 67 | live := hostCertPEM(t, "a.example.com", later) |
| 68 | putLocal(t, local, "a.example.com", live) |
| 69 | putLocal(t, local, "a.example.com+rsa", hostCertPEM(t, "a.example.com", later)) |
| 70 | putLocal(t, local, "wildcard-acct.example.com.pem", hostCertPEM(t, "*.acct.example.com", later)) |
| 71 | putLocal(t, local, "expired.example.com", hostCertPEM(t, "expired.example.com", time.Now().Add(-time.Hour))) |
| 72 | putLocal(t, local, "garbage.example.com", []byte("not a cert")) |
| 73 | putLocal(t, local, "a.example.com+http-01", []byte("token-body")) |
| 74 | putLocal(t, local, "a.example.com+token", hostCertPEM(t, "a.example.com", later)) |
| 75 | // Disk residue that is NOT the deployment's to replicate: a route |
| 76 | // detached long ago, a DirCache temp file a kill left behind, a |
| 77 | // wildcard PEM named for a parent its leaf does not cover. |
| 78 | putLocal(t, local, "gone.example.com", hostCertPEM(t, "gone.example.com", later)) |
| 79 | putLocal(t, local, "a.example.com1762384410", live) |
| 80 | putLocal(t, local, "wildcard-other.example.com.pem", hostCertPEM(t, "*.acct.example.com", later)) |
| 81 | putLocal(t, local, "wildcard-left.example.com.pem", hostCertPEM(t, "*.left.example.com", later)) // parent no longer current |
| 82 | putLocal(t, local, "acme_account+key", []byte("LOCAL-ACCOUNT")) |
| 83 | putLocal(t, local, wildcardAccountKeyName, []byte("LOCAL-WILDCARD-ACCOUNT")) |
| 84 | |
| 85 | m := issuerMirror(t, s, local, true) |
| 86 | m.backfill(context.Background()) |
| 87 | |
| 88 | for _, name := range []string{"a.example.com", "a.example.com+rsa", "wildcard-acct.example.com.pem"} { |
| 89 | if _, ok := bucketHas(t, s, name); !ok { |
| 90 | t.Errorf("%s not backfilled", name) |
| 91 | } |
| 92 | } |
| 93 | if got, _ := bucketHas(t, s, "a.example.com"); string(got) != string(live) { |
| 94 | t.Errorf("a.example.com body differs from the local cache") |
| 95 | } |
| 96 | for _, name := range []string{"expired.example.com", "garbage.example.com", "a.example.com+http-01", "a.example.com+token", |
| 97 | "gone.example.com", "a.example.com1762384410", "wildcard-other.example.com.pem", "wildcard-left.example.com.pem"} { |
| 98 | if _, ok := bucketHas(t, s, name); ok { |
| 99 | t.Errorf("%s must not be published", name) |
| 100 | } |
| 101 | } |
| 102 | // Account keys: the bucket had none, so this box's become the |
| 103 | // deployment's. |
| 104 | for name, want := range map[string]string{"acme_account+key": "LOCAL-ACCOUNT", wildcardAccountKeyName: "LOCAL-WILDCARD-ACCOUNT"} { |
| 105 | if got, ok := bucketHas(t, s, name); !ok || string(got) != want { |
| 106 | t.Errorf("%s: got %q, %v; want seeded", name, got, ok) |
| 107 | } |
| 108 | } |
| 109 | |
| 110 | // Steady state: a second pass after a poll writes nothing. A fresh |
| 111 | // store wrapper counts. |
| 112 | if err := m.Refresh(context.Background()); err != nil { |
| 113 | t.Fatal(err) |
| 114 | } |
| 115 | c := &putCountingStore{Store: s} |
| 116 | m.s = c |
| 117 | m.backfill(context.Background()) |
| 118 | if c.puts != 0 { |
| 119 | t.Fatalf("second pass published %d entries; want 0", c.puts) |
| 120 | } |
| 121 | } |
| 122 | |
| 123 | type putCountingStore struct { |
| 124 | release.Store |
| 125 | puts int |
| 126 | } |
| 127 | |
| 128 | func (c *putCountingStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) { |
| 129 | c.puts++ |
| 130 | return c.Store.Put(ctx, key, body, ifMatch) |
| 131 | } |
| 132 | |
| 133 | // #318: certs publish from ANY agent-run box, lease or no lease — the |
| 134 | // rung-1 rehearsal found the issuer-only rule left every cert minted on a |
| 135 | // box that never held the lease sitting on that box's disk alone, exactly |
| 136 | // the bytes a failover loses. Only the agent-run predicate bars a mirror: |
| 137 | // a hand-run edge, and a mirror built without an Edge, still never publish. |
| 138 | func TestBackfillRunsOnANonIssuer(t *testing.T) { |
| 139 | log.SetOutput(io.Discard) |
| 140 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 141 | s := release.NewDirStore(t.TempDir()) |
| 142 | local := autocert.DirCache(t.TempDir()) |
| 143 | putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour))) |
| 144 | |
| 145 | m2 := newCertMirror(s, local) // mayBackfill nil: no Edge, never publishes |
| 146 | m2.backfill(context.Background()) |
| 147 | if _, ok := bucketHas(t, s, "a.example.com"); ok { |
| 148 | t.Fatal("a gate-less mirror published") |
| 149 | } |
| 150 | |
| 151 | m := issuerMirror(t, s, local, false) // agent-run, NOT holding the lease |
| 152 | m.backfill(context.Background()) |
| 153 | if _, ok := bucketHas(t, s, "a.example.com"); !ok { |
| 154 | t.Fatal("an agent-run non-issuer did not publish its own cert") |
| 155 | } |
| 156 | |
| 157 | // The Edge wiring: eligibility is issuer-gate present AND contending |
| 158 | // in the election; a store-carrying edge with no agent (hand-run) may |
| 159 | // never backfill, and neither may an agent-run box whose housekeeper |
| 160 | // mode is "never" (the grandfathered fleet box). |
| 161 | e := &Edge{staticStore: s} |
| 162 | m3 := e.newCertMirror(local) |
| 163 | if m3.mayBackfill() { |
| 164 | t.Fatal("a hand-run edge may backfill") |
| 165 | } |
| 166 | if m3.issuing() { |
| 167 | t.Fatal("an ungated edge counts as the issuer") |
| 168 | } |
| 169 | e.issuer = func() (context.Context, bool) { return context.Background(), false } |
| 170 | if m3.mayBackfill() { |
| 171 | t.Fatal("an agent-run box that does not contend may backfill (fleet box)") |
| 172 | } |
| 173 | e.contends = func() bool { return false } |
| 174 | if m3.mayBackfill() { |
| 175 | t.Fatal("a contends()==false box may backfill (fleet box)") |
| 176 | } |
| 177 | e.contends = func() bool { return true } |
| 178 | if !m3.mayBackfill() { |
| 179 | t.Fatal("an agent-run contender that does not hold the lease may not backfill") |
| 180 | } |
| 181 | if m3.issuing() { |
| 182 | t.Fatal("a non-holding edge counts as the issuer") |
| 183 | } |
| 184 | e.issuer = func() (context.Context, bool) { return context.Background(), true } |
| 185 | if !m3.issuing() { |
| 186 | t.Fatal("a holding edge is not the issuer") |
| 187 | } |
| 188 | // And end-to-end through backfill itself: the hand-run shape (gate |
| 189 | // present through serving-side wiring is irrelevant — no contender |
| 190 | // predicate) walks nothing and publishes nothing. |
| 191 | e2 := &Edge{staticStore: s} |
| 192 | putLocal(t, local, "handrun.example.com", hostCertPEM(t, "handrun.example.com", time.Now().Add(time.Hour))) |
| 193 | m4 := e2.newCertMirror(local) |
| 194 | m4.allowed = func(string) bool { return true } |
| 195 | m4.backfill(context.Background()) |
| 196 | if _, ok := bucketHas(t, s, "handrun.example.com"); ok { |
| 197 | t.Fatal("a hand-run edge published through backfill") |
| 198 | } |
| 199 | } |
| 200 | |
| 201 | // Seeding the deployment's ACME identity stays single-writer: a non-issuer |
| 202 | // never offers its own key to an empty bucket — and the unseeded key is NOT |
| 203 | // settled, so the same box publishes it on the first pass where it holds. |
| 204 | func TestBackfillAccountKeySeedsOnlyFromTheIssuer(t *testing.T) { |
| 205 | log.SetOutput(io.Discard) |
| 206 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 207 | s := release.NewDirStore(t.TempDir()) |
| 208 | local := autocert.DirCache(t.TempDir()) |
| 209 | putLocal(t, local, "acme_account+key", []byte("MINE")) |
| 210 | |
| 211 | held := false |
| 212 | m := issuerMirror(t, s, local, false) |
| 213 | m.issuing = func() bool { return held } |
| 214 | m.backfill(context.Background()) |
| 215 | if _, ok := bucketHas(t, s, "acme_account+key"); ok { |
| 216 | t.Fatal("a non-issuer seeded the account key") |
| 217 | } |
| 218 | held = true // the same process later wins the lease: same stamps, same memo |
| 219 | m.backfill(context.Background()) |
| 220 | if got, _ := bucketHas(t, s, "acme_account+key"); string(got) != "MINE" { |
| 221 | t.Fatalf("the new issuer did not seed the account key it holds: %q", got) |
| 222 | } |
| 223 | } |
| 224 | |
| 225 | // #318's repair loop: another box clobbers the bucket with an OLDER leaf |
| 226 | // after this box settled against the newer one. The stamp carries the |
| 227 | // bucket copy's identity, so the settled box un-settles on its next poll |
| 228 | // and republishes the newer cert — the clobber is transient, not durable. |
| 229 | func TestBackfillRepairsAnOlderLeafClobber(t *testing.T) { |
| 230 | var buf strings.Builder |
| 231 | log.SetOutput(&buf) |
| 232 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 233 | s := release.NewDirStore(t.TempDir()) |
| 234 | local := autocert.DirCache(t.TempDir()) |
| 235 | newer := hostCertPEM(t, "a.example.com", time.Now().Add(48*time.Hour)) |
| 236 | older := hostCertPEM(t, "a.example.com", time.Now().Add(1*time.Hour)) |
| 237 | putLocal(t, local, "a.example.com", newer) |
| 238 | |
| 239 | m := issuerMirror(t, s, local, false) |
| 240 | m.backfill(context.Background()) |
| 241 | if got, _ := bucketHas(t, s, "a.example.com"); !bytes.Equal(got, newer) { |
| 242 | t.Fatal("setup: newer cert not published") |
| 243 | } |
| 244 | if err := m.Refresh(context.Background()); err != nil { |
| 245 | t.Fatal(err) |
| 246 | } |
| 247 | m.backfill(context.Background()) // settles: disk == bucket == newer |
| 248 | if !strings.Contains(buf.String(), "backfilled 1 cache entry") { |
| 249 | t.Fatalf("the publishing pass did not report itself:\n%s", buf.String()) |
| 250 | } |
| 251 | |
| 252 | // Another box lands the older leaf (its view was stale). |
| 253 | putBucket(t, s, "a.example.com", older) |
| 254 | if err := m.Refresh(context.Background()); err != nil { |
| 255 | t.Fatal(err) |
| 256 | } |
| 257 | m.backfill(context.Background()) |
| 258 | if got, _ := bucketHas(t, s, "a.example.com"); !bytes.Equal(got, newer) { |
| 259 | t.Fatal("the settled box did not repair an older-leaf clobber") |
| 260 | } |
| 261 | } |
| 262 | |
| 263 | // A wildcard PEM on a non-issuer's disk publishes like any owned cert — |
| 264 | // bounded by the manager's parent set exactly as on the issuer. |
| 265 | func TestBackfillWildcardFromANonIssuer(t *testing.T) { |
| 266 | log.SetOutput(io.Discard) |
| 267 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 268 | s := release.NewDirStore(t.TempDir()) |
| 269 | local := autocert.DirCache(t.TempDir()) |
| 270 | putLocal(t, local, "wildcard-acct.example.com.pem", hostCertPEM(t, "*.acct.example.com", time.Now().Add(time.Hour))) |
| 271 | putLocal(t, local, "wildcard-other.example.com.pem", hostCertPEM(t, "*.other.example.com", time.Now().Add(time.Hour))) |
| 272 | |
| 273 | m := issuerMirror(t, s, local, false) |
| 274 | m.backfill(context.Background()) |
| 275 | if _, ok := bucketHas(t, s, "wildcard-acct.example.com.pem"); !ok { |
| 276 | t.Fatal("a non-issuer did not publish the wildcard PEM for a current parent") |
| 277 | } |
| 278 | if _, ok := bucketHas(t, s, "wildcard-other.example.com.pem"); ok { |
| 279 | t.Fatal("a wildcard PEM outside the parent set published") |
| 280 | } |
| 281 | } |
| 282 | |
| 283 | // The other cert shapes ride the same freshness path as plain hosts. |
| 284 | func TestBackfillWildcardAndRSAThroughTheFreshnessPath(t *testing.T) { |
| 285 | log.SetOutput(io.Discard) |
| 286 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 287 | s := release.NewDirStore(t.TempDir()) |
| 288 | local := autocert.DirCache(t.TempDir()) |
| 289 | soon, later := time.Now().Add(24*time.Hour), time.Now().Add(48*time.Hour) |
| 290 | |
| 291 | putBucket(t, s, "wildcard-acct.example.com.pem", hostCertPEM(t, "*.acct.example.com", soon)) |
| 292 | newWild := hostCertPEM(t, "*.acct.example.com", later) |
| 293 | putLocal(t, local, "wildcard-acct.example.com.pem", newWild) |
| 294 | |
| 295 | putBucket(t, s, "a.example.com+rsa", hostCertPEM(t, "a.example.com", soon)) |
| 296 | newRSA := hostCertPEM(t, "a.example.com", later) |
| 297 | putLocal(t, local, "a.example.com+rsa", newRSA) |
| 298 | |
| 299 | // A bucket object that does not parse is replaced by a live local one. |
| 300 | putBucket(t, s, "b.example.com", []byte("corrupt")) |
| 301 | goodB := hostCertPEM(t, "b.example.com", later) |
| 302 | putLocal(t, local, "b.example.com", goodB) |
| 303 | |
| 304 | m := issuerMirror(t, s, local, true) |
| 305 | m.backfill(context.Background()) |
| 306 | for name, want := range map[string][]byte{"wildcard-acct.example.com.pem": newWild, "a.example.com+rsa": newRSA, "b.example.com": goodB} { |
| 307 | if got, _ := bucketHas(t, s, name); string(got) != string(want) { |
| 308 | t.Errorf("%s: bucket copy not brought forward", name) |
| 309 | } |
| 310 | } |
| 311 | } |
| 312 | |
| 313 | type failingPutStore struct { |
| 314 | release.Store |
| 315 | mu sync.Mutex |
| 316 | puts int |
| 317 | } |
| 318 | |
| 319 | func (f *failingPutStore) Put(context.Context, string, []byte, string) (string, error) { |
| 320 | f.mu.Lock() |
| 321 | defer f.mu.Unlock() |
| 322 | f.puts++ |
| 323 | return "", errors.New("AccessDenied") |
| 324 | } |
| 325 | |
| 326 | // A write-denied bucket retries every pass and says so on the file's |
| 327 | // first-and-every-tenth cadence, never every 30 seconds. |
| 328 | func TestBackfillFailureIsRetriedAndThrottled(t *testing.T) { |
| 329 | var buf strings.Builder |
| 330 | log.SetOutput(&buf) |
| 331 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 332 | inner := release.NewDirStore(t.TempDir()) |
| 333 | local := autocert.DirCache(t.TempDir()) |
| 334 | putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour))) |
| 335 | m := issuerMirror(t, inner, local, true) |
| 336 | f := &failingPutStore{Store: inner} |
| 337 | m.s = f |
| 338 | for i := 0; i < 12; i++ { |
| 339 | m.backfill(context.Background()) |
| 340 | } |
| 341 | if f.puts != 12 { |
| 342 | t.Fatalf("puts = %d; want one retry per pass", f.puts) |
| 343 | } |
| 344 | if n := strings.Count(buf.String(), "not published"); n != 2 { |
| 345 | t.Fatalf("logged %d times over 12 failing passes; want 2 (#1, #10):\n%s", n, buf.String()) |
| 346 | } |
| 347 | } |
| 348 | |
| 349 | // Steady state does not re-read the disk: an entry whose stamp has not |
| 350 | // moved since it matched the bucket is skipped on the listing alone. |
| 351 | func TestBackfillSteadyStateDoesNotReread(t *testing.T) { |
| 352 | log.SetOutput(io.Discard) |
| 353 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 354 | s := release.NewDirStore(t.TempDir()) |
| 355 | local := autocert.DirCache(t.TempDir()) |
| 356 | putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour))) |
| 357 | m := issuerMirror(t, s, local, true) |
| 358 | m.backfill(context.Background()) |
| 359 | if err := m.Refresh(context.Background()); err != nil { |
| 360 | t.Fatal(err) |
| 361 | } |
| 362 | // Residue the pass settles as not-owned must not be re-read either. |
| 363 | putLocal(t, local, "gone.example.com", hostCertPEM(t, "gone.example.com", time.Now().Add(time.Hour))) |
| 364 | m.backfill(context.Background()) |
| 365 | if os.Getuid() == 0 { |
| 366 | t.Skip("root ignores mode bits") |
| 367 | } |
| 368 | // Listing and stat still work; any read now fails loudly. |
| 369 | for _, name := range []string{"a.example.com", "gone.example.com"} { |
| 370 | if err := os.Chmod(filepath.Join(string(local), name), 0o000); err != nil { |
| 371 | t.Fatal(err) |
| 372 | } |
| 373 | } |
| 374 | var buf strings.Builder |
| 375 | log.SetOutput(&buf) |
| 376 | c := &putCountingStore{Store: s} |
| 377 | m.s = c |
| 378 | m.backfill(context.Background()) |
| 379 | if c.puts != 0 { |
| 380 | t.Fatalf("steady state published %d", c.puts) |
| 381 | } |
| 382 | if strings.Contains(buf.String(), "not published") { |
| 383 | t.Fatalf("steady state re-read the disk:\n%s", buf.String()) |
| 384 | } |
| 385 | // And a moved stamp IS re-read: touch one and the read fails visibly. |
| 386 | now := time.Now().Add(time.Second) |
| 387 | if err := os.Chtimes(filepath.Join(string(local), "a.example.com"), now, now); err != nil { |
| 388 | t.Fatal(err) |
| 389 | } |
| 390 | m.backfill(context.Background()) |
| 391 | if !strings.Contains(buf.String(), "not published") { |
| 392 | t.Fatal("a changed stamp was not re-read") |
| 393 | } |
| 394 | } |
| 395 | |
| 396 | // autocert writes the cache while the backfill walks it. Whatever the |
| 397 | // interleaving, nothing but complete, owned entries reach the bucket. |
| 398 | func TestBackfillRacesAConcurrentIssuance(t *testing.T) { |
| 399 | log.SetOutput(io.Discard) |
| 400 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 401 | s := release.NewDirStore(t.TempDir()) |
| 402 | local := autocert.DirCache(t.TempDir()) |
| 403 | later := time.Now().Add(time.Hour) |
| 404 | pems := map[string][]byte{} |
| 405 | for i := 0; i < 8; i++ { |
| 406 | h := string(rune('a'+i)) + ".example.com" |
| 407 | pems[h] = hostCertPEM(t, h, later) |
| 408 | } |
| 409 | m := issuerMirror(t, s, local, true) |
| 410 | var wg sync.WaitGroup |
| 411 | wg.Add(2) |
| 412 | go func() { |
| 413 | defer wg.Done() |
| 414 | for h, pem := range pems { |
| 415 | putLocal(t, local, h, pem) |
| 416 | } |
| 417 | }() |
| 418 | go func() { |
| 419 | defer wg.Done() |
| 420 | for i := 0; i < 20; i++ { |
| 421 | m.backfill(context.Background()) |
| 422 | } |
| 423 | }() |
| 424 | wg.Wait() |
| 425 | m.backfill(context.Background()) |
| 426 | keys, _ := s.List(context.Background(), certPrefix) |
| 427 | for _, k := range keys { |
| 428 | name := strings.TrimPrefix(k, certPrefix) |
| 429 | want, ok := pems[name] |
| 430 | if !ok { |
| 431 | t.Errorf("stray bucket key %s", k) |
| 432 | continue |
| 433 | } |
| 434 | if got, _ := bucketHas(t, s, name); string(got) != string(want) { |
| 435 | t.Errorf("%s: bucket body is not the cache's", name) |
| 436 | } |
| 437 | } |
| 438 | if len(keys) != len(pems) { |
| 439 | t.Fatalf("bucket has %d of %d", len(keys), len(pems)) |
| 440 | } |
| 441 | } |
| 442 | |
| 443 | // The renewed-before-the-gate case: the bucket's copy is older than the |
| 444 | // disk's, so the disk wins; and the reverse — the bucket is fresher — leaves |
| 445 | // the bucket alone (the mirror will bring the disk up instead). |
| 446 | func TestBackfillReplacesOnlyAnOlderBucketCert(t *testing.T) { |
| 447 | log.SetOutput(io.Discard) |
| 448 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 449 | s := release.NewDirStore(t.TempDir()) |
| 450 | local := autocert.DirCache(t.TempDir()) |
| 451 | soon := time.Now().Add(24 * time.Hour) |
| 452 | later := time.Now().Add(48 * time.Hour) |
| 453 | |
| 454 | oldPEM := hostCertPEM(t, "old.example.com", soon) |
| 455 | newPEM := hostCertPEM(t, "old.example.com", later) |
| 456 | putBucket(t, s, "old.example.com", oldPEM) |
| 457 | putLocal(t, local, "old.example.com", newPEM) |
| 458 | |
| 459 | freshBucket := hostCertPEM(t, "fresh.example.com", later) |
| 460 | putBucket(t, s, "fresh.example.com", freshBucket) |
| 461 | putLocal(t, local, "fresh.example.com", hostCertPEM(t, "fresh.example.com", soon)) |
| 462 | |
| 463 | m := issuerMirror(t, s, local, true) |
| 464 | m.backfill(context.Background()) |
| 465 | |
| 466 | if got, _ := bucketHas(t, s, "old.example.com"); string(got) != string(newPEM) { |
| 467 | t.Error("an older bucket cert was not replaced by the disk's newer one") |
| 468 | } |
| 469 | if got, _ := bucketHas(t, s, "fresh.example.com"); string(got) != string(freshBucket) { |
| 470 | t.Error("a fresher bucket cert was clobbered by an older local one") |
| 471 | } |
| 472 | } |
| 473 | |
| 474 | // The deployment's ACME identity is whichever key the bucket already |
| 475 | // holds — a takeover issuer's own key must never displace it. |
| 476 | func TestBackfillNeverReplacesABucketAccountKey(t *testing.T) { |
| 477 | s := release.NewDirStore(t.TempDir()) |
| 478 | local := autocert.DirCache(t.TempDir()) |
| 479 | putBucket(t, s, "acme_account+key", []byte("DEPLOYMENT")) |
| 480 | putBucket(t, s, wildcardAccountKeyName, []byte("DEPLOYMENT-WILDCARD")) |
| 481 | // Refresh seeds the bucket keys locally only when absent; make the local |
| 482 | // ones differ first. |
| 483 | putLocal(t, local, "acme_account+key", []byte("MINE")) |
| 484 | putLocal(t, local, wildcardAccountKeyName, []byte("MINE-WILDCARD")) |
| 485 | |
| 486 | m := issuerMirror(t, s, local, true) |
| 487 | m.backfill(context.Background()) |
| 488 | if got, _ := bucketHas(t, s, "acme_account+key"); string(got) != "DEPLOYMENT" { |
| 489 | t.Fatalf("account key replaced: %q", got) |
| 490 | } |
| 491 | if got, _ := bucketHas(t, s, wildcardAccountKeyName); string(got) != "DEPLOYMENT-WILDCARD" { |
| 492 | t.Fatalf("wildcard account key replaced: %q", got) |
| 493 | } |
| 494 | } |
| 495 | |
| 496 | // Only an on-disk cache can be enumerated; anything else is a quiet no-op, |
| 497 | // never a panic in the poll loop. |
| 498 | func TestBackfillNeedsADirCache(t *testing.T) { |
| 499 | s := release.NewDirStore(t.TempDir()) |
| 500 | m := issuerMirror(t, s, nil, true) |
| 501 | m.backfill(context.Background()) |
| 502 | m2 := issuerMirror(t, s, newCountingCache(t), true) |
| 503 | m2.backfill(context.Background()) |
| 504 | m3 := issuerMirror(t, s, autocert.DirCache(t.TempDir()+"/missing"), true) |
| 505 | m3.backfill(context.Background()) |
| 506 | if keys, _ := s.List(context.Background(), certPrefix); len(keys) != 0 { |
| 507 | t.Fatalf("published %v", keys) |
| 508 | } |
| 509 | } |
| 510 | |
| 511 | // The poll loop runs the backfill after each successful refresh: an issuer |
| 512 | // that boots with a full cache and an empty bucket fills it without waiting |
| 513 | // for a renewal. |
| 514 | func TestCertMirrorLoopBackfillsOnTheIssuer(t *testing.T) { |
| 515 | prev := certMirrorInterval |
| 516 | certMirrorInterval = 5 * time.Millisecond |
| 517 | t.Cleanup(func() { certMirrorInterval = prev }) |
| 518 | log.SetOutput(io.Discard) |
| 519 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 520 | |
| 521 | s := release.NewDirStore(t.TempDir()) |
| 522 | local := autocert.DirCache(t.TempDir()) |
| 523 | putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour))) |
| 524 | m := newCertMirror(s, local) |
| 525 | m.mayBackfill = func() bool { return true } |
| 526 | m.issuing = func() bool { return true } |
| 527 | m.allowed = func(string) bool { return true } |
| 528 | stop := make(chan struct{}) |
| 529 | done := make(chan struct{}) |
| 530 | go func() { m.run(stop); close(done) }() |
| 531 | deadline := time.Now().Add(2 * time.Second) |
| 532 | for { |
| 533 | if _, ok := bucketHas(t, s, "a.example.com"); ok { |
| 534 | break |
| 535 | } |
| 536 | if time.Now().After(deadline) { |
| 537 | t.Fatal("the loop never backfilled") |
| 538 | } |
| 539 | time.Sleep(2 * time.Millisecond) |
| 540 | } |
| 541 | // And the mirror then serves what it published. |
| 542 | deadline = time.Now().Add(2 * time.Second) |
| 543 | for { |
| 544 | if c, ok := m.Cert("a.example.com"); ok && c != nil { |
| 545 | break |
| 546 | } |
| 547 | if time.Now().After(deadline) { |
| 548 | t.Fatal("the mirror never served the backfilled cert") |
| 549 | } |
| 550 | time.Sleep(2 * time.Millisecond) |
| 551 | } |
| 552 | close(stop) |
| 553 | <-done |
| 554 | } |
| 555 | |
| 556 | // The bucket side is part of the stamp: an entry settled as in sync that |
| 557 | // later leaves the bucket (a prune, a mis-delete) republishes on the next |
| 558 | // pass even though the file has not moved — otherwise #291's failure mode |
| 559 | // returns by a different door. |
| 560 | func TestBackfillRepublishesAnEntryThatLeftTheBucket(t *testing.T) { |
| 561 | log.SetOutput(io.Discard) |
| 562 | t.Cleanup(func() { log.SetOutput(os.Stderr) }) |
| 563 | s := release.NewDirStore(t.TempDir()) |
| 564 | local := autocert.DirCache(t.TempDir()) |
| 565 | putLocal(t, local, "a.example.com", hostCertPEM(t, "a.example.com", time.Now().Add(time.Hour))) |
| 566 | ctx := context.Background() |
| 567 | m := issuerMirror(t, s, local, true) |
| 568 | m.backfill(ctx) |
| 569 | if err := m.Refresh(ctx); err != nil { |
| 570 | t.Fatal(err) |
| 571 | } |
| 572 | m.backfill(ctx) // settles in sync |
| 573 | if err := s.Delete(ctx, certKey("a.example.com")); err != nil { |
| 574 | t.Fatal(err) |
| 575 | } |
| 576 | if err := m.Refresh(ctx); err != nil { |
| 577 | t.Fatal(err) |
| 578 | } |
| 579 | m.backfill(ctx) |
| 580 | if _, ok := bucketHas(t, s, "a.example.com"); !ok { |
| 581 | t.Fatal("an entry that left the bucket was never republished") |
| 582 | } |
| 583 | } |
| 584 | |
| 585 | // The production wiring: the wildcard manager's parent set reaches the |
| 586 | // mirror, or every wildcard PEM would silently never backfill. |
| 587 | func TestAttachWildcardToMirrorWiresParentsAndWake(t *testing.T) { |
| 588 | w := &wildcardManager{parents: []string{"acct.example.com"}} |
| 589 | m := newCertMirror(release.NewDirStore(t.TempDir()), nil) |
| 590 | attachWildcardToMirror(m, w) |
| 591 | if m.wildcardParents == nil || !slices.Equal(m.wildcardParents(), []string{"acct.example.com"}) { |
| 592 | t.Fatal("wildcardParents not wired to the manager's current set") |
| 593 | } |
| 594 | if m.wake == nil { |
| 595 | t.Fatal("wake not wired") |
| 596 | } |
| 597 | attachWildcardToMirror(nil, w) // no mirror: no panic |
| 598 | attachWildcardToMirror(m, nil) // no manager: leaves the wiring alone |
| 599 | if m.wildcardParents == nil { |
| 600 | t.Fatal("a nil manager cleared the wiring") |
| 601 | } |
| 602 | } |
| 603 | |
amadan