carlos / platform Public

Download this file

1	package edge
2
3	import (
4	"bytes"
5	"context"
6	"crypto/tls"
7	"errors"
8	"fmt"
9	"io"
10	"log"
11	"net/http"
12	"net/http/httptest"
13	"os"
14	"runtime"
15	"strings"
16	"sync"
17	"sync/atomic"
18	"testing"
19	"time"
20
21	"golang.org/x/crypto/acme"
22	"golang.org/x/crypto/acme/autocert"
23
24	"github.com/carlosframework/platform/internal/release"
25	)
26
27	// certStoreGet is the two-value read the tests want: the Store's third
28	// return (the etag) is never interesting here.
29	func certStoreGet(t *testing.T, s release.Store, key string) ([]byte, error) {
30	t.Helper()
31	b, _, err := s.Get(context.Background(), key)
32	return b, err
33	}
34
35	func TestCertKeyShape(t *testing.T) {
36	// Pinned: Task 5's mirror lists this exact prefix and Task 6's door ACL
37	// test names these exact keys. autocert's cache names go through
38	// UNCHANGED — per-host PEM, the account key, HTTP-01 tokens — and so do
39	// the wildcard manager's (wildcard.go), which is what lets one mirror of
40	// one prefix rebuild a whole cache.
41	for name, want := range map[string]string{
42	"example.com": "control/certs/example.com",
43	"example.com+rsa": "control/certs/example.com+rsa",
44	"acme_account+key": "control/certs/acme_account+key",
45	"tok+http-01": "control/certs/tok+http-01",
46	"wildcard-go.tito.io.pem": "control/certs/wildcard-go.tito.io.pem",
47	"wildcard-account.key": "control/certs/wildcard-account.key",
48	} {
49	if got := certKey(name); got != want {
50	t.Fatalf("certKey(%q) = %q, want %q", name, got, want)
51	}
52	}
53	}
54
55	func TestPublishingCacheMirrorsPutsToBucket(t *testing.T) {
56	s := release.NewDirStore(t.TempDir())
57	pc := &publishingCache{inner: autocert.DirCache(t.TempDir()), s: s}
58	if err := pc.Put(context.Background(), "example.com", []byte("PEM")); err != nil {
59	t.Fatal(err)
60	}
61	got, err := certStoreGet(t, s, "control/certs/example.com")
62	if err != nil || string(got) != "PEM" {
63	t.Fatalf("bucket copy = %q, %v", got, err)
64	}
65	// Local cache also holds it (autocert's own read path keeps working).
66	if b, err := pc.Get(context.Background(), "example.com"); err != nil || string(b) != "PEM" {
67	t.Fatalf("local copy = %q, %v", b, err)
68	}
69	}
70
71	func TestPublishingCacheDeleteRemovesBucketCopy(t *testing.T) {
72	s := release.NewDirStore(t.TempDir())
73	pc := &publishingCache{inner: autocert.DirCache(t.TempDir()), s: s}
74	_ = pc.Put(context.Background(), "tok+http-01", []byte("challenge"))
75	if err := pc.Delete(context.Background(), "tok+http-01"); err != nil {
76	t.Fatal(err)
77	}
78	if _, err := certStoreGet(t, s, "control/certs/tok+http-01"); err == nil {
79	t.Fatal("token must be deleted from bucket after validation")
80	}
81	if _, err := pc.Get(context.Background(), "tok+http-01"); !errors.Is(err, autocert.ErrCacheMiss) {
82	t.Fatalf("local copy after Delete: want ErrCacheMiss, got %v", err)
83	}
84	}
85
86	// A renewal writes the SAME key again. A plain create-only Put would lose
87	// every renewal to a CAS conflict, so the publish is last-writer-wins.
88	func TestPublishingCacheReplacesAnExistingBucketCopy(t *testing.T) {
89	s := release.NewDirStore(t.TempDir())
90	pc := &publishingCache{inner: autocert.DirCache(t.TempDir()), s: s}
91	if err := pc.Put(context.Background(), "example.com", []byte("OLD")); err != nil {
92	t.Fatal(err)
93	}
94	if err := pc.Put(context.Background(), "example.com", []byte("NEW")); err != nil {
95	t.Fatal(err)
96	}
97	got, err := certStoreGet(t, s, "control/certs/example.com")
98	if err != nil || string(got) != "NEW" {
99	t.Fatalf("bucket copy after renewal = %q, %v — the publish must replace, not create-only", got, err)
100	}
101	}
102
103	// failingStore fails every write; reads are the real store's.
104	type failingStore struct {
105	release.Store
106	mu sync.Mutex
107	puts int
108	dels int
109	putErr error
110	}
111
112	func (f *failingStore) Put(context.Context, string, []byte, string) (string, error) {
113	f.mu.Lock()
114	defer f.mu.Unlock()
115	f.puts++
116	return "", f.putErr
117	}
118
119	func (f *failingStore) Delete(context.Context, string) error {
120	f.mu.Lock()
121	defer f.mu.Unlock()
122	f.dels++
123	return errors.New("bucket down")
124	}
125
126	// Local issuance must survive a bucket blip: the cert is on disk, the
127	// failure is loud, and autocert never sees an error (a returned error would
128	// abort an issuance that actually succeeded).
129	func TestPublishingCacheBucketFailureNeverFailsTheLocalOperation(t *testing.T) {
130	var buf bytes.Buffer
131	prev := log.Writer()
132	log.SetOutput(&buf)
133	t.Cleanup(func() { log.SetOutput(prev) })
134
135	s := &failingStore{Store: release.NewDirStore(t.TempDir()), putErr: errors.New("bucket down")}
136	dir := t.TempDir()
137	pc := &publishingCache{inner: autocert.DirCache(dir), s: s}
138
139	if err := pc.Put(context.Background(), "example.com", []byte("PEM")); err != nil {
140	t.Fatalf("Put must not fail on a bucket error: %v", err)
141	}
142	if b, err := pc.Get(context.Background(), "example.com"); err != nil || string(b) != "PEM" {
143	t.Fatalf("local copy = %q, %v — local durability comes first", b, err)
144	}
145	if err := pc.Delete(context.Background(), "example.com"); err != nil {
146	t.Fatalf("Delete must not fail on a bucket error: %v", err)
147	}
148	if out := buf.String(); !strings.Contains(out, "certsync: bucket publish failed") ||
149	!strings.Contains(out, "certsync: bucket delete failed") {
150	t.Fatalf("a bucket failure must be loud; log was:\n%s", out)
151	}
152	}
153
154	// The gate is the whole point: only the housekeeper publishes. A non-issuer
155	// keeps its local cache (it must still serve) and writes nothing.
156	func TestPublishingCacheNonIssuerPublishesNothing(t *testing.T) {
157	s := &failingStore{Store: release.NewDirStore(t.TempDir())}
158	pc := &publishingCache{
159	inner: autocert.DirCache(t.TempDir()),
160	s: s,
161	issuer: func() (context.Context, bool) { return nil, false },
162	}
163	if err := pc.Put(context.Background(), "example.com", []byte("PEM")); err != nil {
164	t.Fatal(err)
165	}
166	if err := pc.Delete(context.Background(), "example.com"); err != nil {
167	t.Fatal(err)
168	}
169	s.mu.Lock()
170	defer s.mu.Unlock()
171	if s.puts != 0 || s.dels != 0 {
172	t.Fatalf("non-issuer touched the bucket: %d put(s), %d delete(s)", s.puts, s.dels)
173	}
174	}
175
176	// hangingStore blocks every write until its context ends — a bucket that has
177	// stopped answering rather than one that refuses.
178	type hangingStore struct {
179	release.Store
180	mu sync.Mutex
181	entered int
182	}
183
184	func (h *hangingStore) enter() {
185	h.mu.Lock()
186	h.entered++
187	h.mu.Unlock()
188	}
189
190	func (h *hangingStore) Get(ctx context.Context, _ string) ([]byte, string, error) {
191	h.enter()
192	<-ctx.Done()
193	return nil, "", ctx.Err()
194	}
195
196	func (h *hangingStore) Put(ctx context.Context, _ string, _ []byte, _ string) (string, error) {
197	h.enter()
198	<-ctx.Done()
199	return "", ctx.Err()
200	}
201
202	func (h *hangingStore) Delete(ctx context.Context, _ string) error {
203	h.enter()
204	<-ctx.Done()
205	return ctx.Err()
206	}
207
208	// autocert calls Delete while holding challengeMu for write, and serves
209	// tls-alpn-01 handshakes and :80 challenge requests under that same mutex —
210	// so a publish that waits on a stalled bucket wedges the issuer's front
211	// door. Every bucket call gets its own budget.
212	func TestPublishingCacheBoundsHowLongABucketCanStall(t *testing.T) {
213	prev := certPublishTimeout
214	certPublishTimeout = 40 * time.Millisecond
215	t.Cleanup(func() { certPublishTimeout = prev })
216	log.SetOutput(io.Discard)
217	t.Cleanup(func() { log.SetOutput(os.Stderr) })
218
219	s := &hangingStore{}
220	dir := t.TempDir()
221	pc := &publishingCache{inner: autocert.DirCache(dir), s: s}
222
223	for _, step := range []struct {
224	name string
225	run func() error
226	}{
227	{"Put", func() error { return pc.Put(context.Background(), "tok+http-01", []byte("challenge")) }},
228	{"Delete", func() error { return pc.Delete(context.Background(), "tok+http-01") }},
229	} {
230	start := time.Now()
231	if err := step.run(); err != nil {
232	t.Fatalf("%s on a stalled bucket: %v", step.name, err)
233	}
234	if took := time.Since(start); took > 2*time.Second {
235	t.Fatalf("%s waited %s on a stalled bucket — the publish must carry its own budget", step.name, took)
236	}
237	}
238	if s.entered == 0 {
239	t.Fatal("test bug: the bucket was never called")
240	}
241	// The local half completed both times, which is the half that matters.
242	if _, err := pc.Get(context.Background(), "tok+http-01"); !errors.Is(err, autocert.ErrCacheMiss) {
243	t.Fatalf("local delete did not happen: %v", err)
244	}
245	}
246
247	// ctxAwareStore refuses work on a dead context, the way a real S3 client
248	// does and the local DirStore (which ignores its context) does not.
249	type ctxAwareStore struct{ release.Store }
250
251	func (c ctxAwareStore) Get(ctx context.Context, key string) ([]byte, string, error) {
252	if err := ctx.Err(); err != nil {
253	return nil, "", err
254	}
255	return c.Store.Get(ctx, key)
256	}
257
258	func (c ctxAwareStore) Put(ctx context.Context, key string, body []byte, ifMatch string) (string, error) {
259	if err := ctx.Err(); err != nil {
260	return "", err
261	}
262	return c.Store.Put(ctx, key, body, ifMatch)
263	}
264
265	func (c ctxAwareStore) Delete(ctx context.Context, key string) error {
266	if err := ctx.Err(); err != nil {
267	return err
268	}
269	return c.Store.Delete(ctx, key)
270	}
271
272	// A cancelled order still has to clean up its challenge token: the delete
273	// runs on a context detached from the caller's.
274	func TestPublishingCacheDeletesEvenUnderACancelledCaller(t *testing.T) {
275	s := ctxAwareStore{release.NewDirStore(t.TempDir())}
276	pc := &publishingCache{inner: autocert.DirCache(t.TempDir()), s: s}
277	if err := pc.Put(context.Background(), "tok+http-01", []byte("challenge")); err != nil {
278	t.Fatal(err)
279	}
280	dead, cancel := context.WithCancel(context.Background())
281	cancel()
282	_ = pc.Delete(dead, "tok+http-01")
283	if _, err := certStoreGet(t, s, "control/certs/tok+http-01"); err == nil {
284	t.Fatal("an abandoned order must still take its token out of the bucket")
285	}
286	}
287
288	// ---- the issuance gate ------------------------------------------------------
289
290	func testIssuerEdge(hosts ...string) *Edge {
291	e := &Edge{}
292	m := map[string]upstream{}
293	for _, h := range hosts {
294	m[h] = upstream{}
295	}
296	e.upstreams.Store(&m)
297	return e
298	}
299
300	// seedCachedCert writes a cert for host into cache in autocert's own entry
301	// shape (private key PEM, then the chain) — what a previous process, or Task
302	// 5's mirror, leaves on a box's disk.
303	func seedCachedCert(t *testing.T, cache autocert.Cache, parent, host string) {
304	t.Helper()
305	cert, key := selfSignedWildcard(t, parent, time.Now().Add(60*24*time.Hour))
306	raw, err := encodeWildcardPEM(cert, key)
307	if err != nil {
308	t.Fatal(err)
309	}
310	if err := cache.Put(context.Background(), host, raw); err != nil {
311	t.Fatal(err)
312	}
313	}
314
315	// ecdsaHello is a handshake that supports ECDSA, so autocert looks under the
316	// bare host name rather than "<host>+rsa".
317	func ecdsaHello(host string) *tls.ClientHelloInfo {
318	return &tls.ClientHelloInfo{
319	ServerName: host,
320	SignatureSchemes: []tls.SignatureScheme{tls.ECDSAWithP256AndSHA256},
321	SupportedCurves: []tls.CurveID{tls.CurveP256},
322	CipherSuites: []uint16{tls.TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256},
323	}
324	}
325
326	// offlineManager is a real autocert.Manager pointed at a dead directory, so
327	// a test that accidentally reaches the ordering path fails offline and fast
328	// instead of dialling Let's Encrypt.
329	func offlineManager(t *testing.T, cache autocert.Cache, pol autocert.HostPolicy) *autocert.Manager {
330	return &autocert.Manager{
331	Prompt: autocert.AcceptTOS,
332	Cache: cache,
333	HostPolicy: pol,
334	Client: &acme.Client{DirectoryURL: deadDirectory(t)},
335	}
336	}
337
338	// A box with no deployment store has nothing to publish to AND nothing to
339	// mirror from — fleet boxes (barred from control/certs/** by the door ACL),
340	// boxes with no bucket or dir, and bucket-without-pubkey boxes. Gating those
341	// would leave them permanently certificate-less, because autocert consults
342	// HostPolicy BEFORE its cache lookup: no orders and no serving either.
343	func TestNoStoreEdgeKeepsIssuingAndServingWithTheGateShut(t *testing.T) {
344	const parent, host = "bab.oncarlos.com", "app.bab.oncarlos.com"
345	dir := autocert.DirCache(t.TempDir())
346	seedCachedCert(t, dir, parent, host)
347
348	e := testIssuerEdge(host)
349	e.issuer = func() (context.Context, bool) { return nil, false } // never holds
350
351	cache := e.armCertIssuance(dir) // no staticStore
352	if _, wrapped := cache.(*publishingCache); wrapped {
353	t.Fatal("no deployment store: there is nothing to publish through")
354	}
355	if _, ok := e.issuerGate(); !ok {
356	t.Fatal("the gate must be cleared, so the wildcard path is ungated too")
357	}
358	got, err := offlineManager(t, cache, e.issuerHostPolicy()).GetCertificate(ecdsaHello(host))
359	if err != nil {
360	t.Fatalf("a box with no mirror must keep serving its own cache: %v", err)
361	}
362	if got.Leaf == nil || got.Leaf.VerifyHostname(host) != nil {
363	t.Fatalf("served cert does not cover %s", host)
364	}
365	}
366
367	// The same edge WITH a store stays gated: the refusal is the gate's, and the
368	// cert it declines to serve is sitting right there on disk — which is the
369	// whole reason Task 5's mirror layer has to sit behind this.
370	func TestStoreBackedEdgeStaysGated(t *testing.T) {
371	const parent, host = "bab.oncarlos.com", "app.bab.oncarlos.com"
372	dir := autocert.DirCache(t.TempDir())
373	seedCachedCert(t, dir, parent, host)
374
375	e := testIssuerEdge(host)
376	e.staticStore = release.NewDirStore(t.TempDir())
377	e.issuer = func() (context.Context, bool) { return nil, false }
378
379	cache := e.armCertIssuance(dir)
380	if _, wrapped := cache.(*publishingCache); !wrapped {
381	t.Fatal("a deployment store means every artifact publishes")
382	}
383	if _, err := cache.Get(context.Background(), host); err != nil {
384	t.Fatalf("test bug: the local cert should still be readable: %v", err)
385	}
386	_, err := offlineManager(t, cache, e.issuerHostPolicy()).GetCertificate(ecdsaHello(host))
387	if !errors.Is(err, errNotIssuer) {
388	t.Fatalf("non-issuer with a mirror behind it = %v, want errNotIssuer", err)
389	}
390	}
391
392	func TestHostPolicyUngatedIsTodaysBehaviour(t *testing.T) {
393	e := testIssuerEdge("app.bab.oncarlos.com")
394	pol := e.issuerHostPolicy()
395	if err := pol(context.Background(), "app.bab.oncarlos.com"); err != nil {
396	t.Fatalf("no gate configured (carlos edge, single box): want issuance allowed, got %v", err)
397	}
398	if err := pol(context.Background(), "stranger.example"); err == nil {
399	t.Fatal("unknown host must still be refused")
400	} else if errors.Is(err, errNotIssuer) {
401	t.Fatalf("unknown host must be refused as unknown, not as not-issuer: %v", err)
402	}
403	}
404
405	func TestHostPolicyRefusesOrdersOnANonIssuer(t *testing.T) {
406	e := testIssuerEdge("app.bab.oncarlos.com")
407	e.issuer = func() (context.Context, bool) { return nil, false }
408	err := e.issuerHostPolicy()(context.Background(), "app.bab.oncarlos.com")
409	if !errors.Is(err, errNotIssuer) {
410	t.Fatalf("non-issuer HostPolicy = %v, want errNotIssuer", err)
411	}
412	}
413
414	func TestHostPolicyAllowsOrdersWhileHolding(t *testing.T) {
415	e := testIssuerEdge("app.bab.oncarlos.com")
416	e.issuer = func() (context.Context, bool) { return context.Background(), true }
417	if err := e.issuerHostPolicy()(context.Background(), "app.bab.oncarlos.com"); err != nil {
418	t.Fatalf("issuer HostPolicy = %v, want nil", err)
419	}
420	}
421
422	// ---- the wildcard (DNS-01) gate --------------------------------------------
423
424	// countingCache counts Gets per key — the wildcard order path's first cache
425	// read is its ACME account key, so "did an order start?" is observable
426	// without a CA.
427	type countingCache struct {
428	autocert.Cache
429	mu sync.Mutex
430	gets map[string]int
431	}
432
433	func newCountingCache(t *testing.T) *countingCache {
434	return &countingCache{Cache: autocert.DirCache(t.TempDir()), gets: map[string]int{}}
435	}
436
437	func (c *countingCache) Get(ctx context.Context, key string) ([]byte, error) {
438	c.mu.Lock()
439	c.gets[key]++
440	c.mu.Unlock()
441	return c.Cache.Get(ctx, key)
442	}
443
444	func (c *countingCache) count(key string) int {
445	c.mu.Lock()
446	defer c.mu.Unlock()
447	return c.gets[key]
448	}
449
450	// deadDirectory is an ACME directory URL nothing is listening on, so an
451	// order that DOES start fails immediately and offline.
452	func deadDirectory(t *testing.T) string {
453	srv := httptest.NewServer(nil)
454	url := srv.URL + "/directory"
455	srv.Close()
456	return url
457	}
458
459	func wildcardGateManager(t *testing.T, cache autocert.Cache) *wildcardManager {
460	w := newWildcardManager([]string{"go.tito.io"}, "", func() []string { return nil }, nil, "", cache, &fakeDNS{})
461	w.directory = deadDirectory(t)
462	w.refresh()
463	return w
464	}
465
466	func TestWildcardPassDoesNotOrderOnANonIssuer(t *testing.T) {
467	cache := newCountingCache(t)
468	w := wildcardGateManager(t, cache)
469	w.issuer = func() (context.Context, bool) { return nil, false }
470
471	// A stood-down pass comes back within the hour: the next one is what
472	// loads the issuer's wildcard out of this box's cache and into memory.
473	if delay := w.pass(); delay > time.Hour {
474	t.Fatalf("non-issuer pass delay = %s, want <= 1h", delay)
475	}
476
477	if n := cache.count(wildcardAccountKeyName); n != 0 {
478	t.Fatalf("a non-issuer started an ACME order (%d account-key read(s))", n)
479	}
480	if w.backingOff("go.tito.io", time.Now()) {
481	t.Fatal("standing down is not a failure: it must not arm the per-parent backoff")
482	}
483	}
484
485	func TestWildcardPassOrdersWhileHolding(t *testing.T) {
486	cache := newCountingCache(t)
487	w := wildcardGateManager(t, cache)
488	w.issuer = func() (context.Context, bool) { return context.Background(), true }
489
490	w.pass()
491
492	if n := cache.count(wildcardAccountKeyName); n == 0 {
493	t.Fatal("the issuer must attempt the order (no account-key read happened)")
494	}
495	if !w.backingOff("go.tito.io", time.Now()) {
496	t.Fatal("a failed order must arm the per-parent backoff")
497	}
498	}
499
500	// The order runs under the HOLDING's context, so a stand-down mid-order is
501	// observable inside the order rather than at the next 12h tick.
502	func TestWildcardOrderRunsUnderTheHoldingContext(t *testing.T) {
503	cache := newCountingCache(t)
504	w := wildcardGateManager(t, cache)
505	held, standDown := context.WithCancel(context.Background())
506	w.issuer = func() (context.Context, bool) { return held, true }
507	standDown()
508
509	w.pass()
510
511	// The account key read is the first thing obtain does; with the holding
512	// already cancelled the order must not have reached the CA. The cache is
513	// context-free, so what this pins is that the cancelled context is the
514	// one the order carries: obtain fails before any network call.
515	if err := held.Err(); err == nil {
516	t.Fatal("test bug: the holding should be cancelled")
517	}
518	w.mu.Lock()
519	got := w.certs["go.tito.io"]
520	w.mu.Unlock()
521	if got != nil {
522	t.Fatal("an order under a cancelled holding must not produce a cert")
523	}
524	}
525
526	func TestWildcardUngatedOrdersAsToday(t *testing.T) {
527	cache := newCountingCache(t)
528	w := wildcardGateManager(t, cache) // no issuer gate at all
529	w.pass()
530	if n := cache.count(wildcardAccountKeyName); n == 0 {
531	t.Fatal("with no gate configured (carlos edge, single box) issuance must run exactly as today")
532	}
533	}
534
535	// ---- the mirror (Task 5) ----------------------------------------------------
536
537	// hostCertPEM mints a cert for host in autocert's own cache-entry shape (the
538	// private key PEM, then the chain) — byte-for-byte what the issuer publishes
539	// to control/certs/<host>.
540	func hostCertPEM(t *testing.T, host string, notAfter time.Time) []byte {
541	t.Helper()
542	cert, key := selfSignedLeaf(t, host, notAfter)
543	raw, err := encodeWildcardPEM(cert, key)
544	if err != nil {
545	t.Fatal(err)
546	}
547	return raw
548	}
549
550	// putBucket writes one control/certs/ entry the way the issuer's
551	// publishingCache would.
552	func putBucket(t *testing.T, s release.Store, name string, body []byte) {
553	t.Helper()
554	if err := putCert(context.Background(), s, name, body); err != nil {
555	t.Fatal(err)
556	}
557	}
558
559	func TestCertMirrorServesBucketCert(t *testing.T) {
560	s := release.NewDirStore(t.TempDir())
561	putBucket(t, s, "example.com", []byte("PEM"))
562	m := newCertMirror(s, nil)
563	if err := m.Refresh(context.Background()); err != nil {
564	t.Fatal(err)
565	}
566	if b, ok := m.Cert("example.com"); !ok || string(b) != "PEM" {
567	t.Fatalf("mirror cert = %q, %v", b, ok)
568	}
569	if _, ok := m.Cert("nobody.example"); ok {
570	t.Fatal("a name the bucket does not hold must miss")
571	}
572	}
573
574	// The HTTP-01 cache key is autocert's, read from the vendored source
575	// (autocert.go:896): path.Base(tokenPath) + "+http-01" — a FLAT name, no
576	// /.well-known/ prefix (publishableCertName would refuse one anyway). Task 7
577	// fails loudly if this drifts.
578	func TestCertMirrorServesChallengeTokens(t *testing.T) {
579	const token = "0123456789abcdefghij0123456789abcdefghijkl"
580	s := release.NewDirStore(t.TempDir())
581	putBucket(t, s, token+"+http-01", []byte("resp"))
582	m := newCertMirror(s, nil)
583	if err := m.Refresh(context.Background()); err != nil {
584	t.Fatal(err)
585	}
586	if b, ok := m.Challenge(token); !ok || string(b) != "resp" {
587	t.Fatalf("challenge = %q, %v", b, ok)
588	}
589	}
590
591	// Account keys are ISSUER material: they must never enter the serving map,
592	// where a name lookup could hand one to a handshake.
593	func TestCertMirrorKeepsAccountKeysOutOfTheServingMap(t *testing.T) {
594	s := release.NewDirStore(t.TempDir())
595	putBucket(t, s, "acme_account+key", []byte("KEY"))
596	putBucket(t, s, wildcardAccountKeyName, []byte("KEY"))
597	putBucket(t, s, "example.com", []byte("PEM"))
598	m := newCertMirror(s, nil)
599	if err := m.Refresh(context.Background()); err != nil {
600	t.Fatal(err)
601	}
602	for _, name := range []string{"acme_account+key", wildcardAccountKeyName} {
603	if _, ok := m.Cert(name); ok {
604	t.Fatalf("%s is issuer material and must not be servable", name)
605	}
606	}
607	if _, ok := m.Cert("example.com"); !ok {
608	t.Fatal("ordinary cert material must still be served")
609	}
610	}
611
612	// listFailingStore fails List (and optionally Get) on demand — a bucket blip
613	// mid-poll.
614	type listFailingStore struct {
615	release.Store
616	mu sync.Mutex
617	fail bool
618	}
619
620	func (l *listFailingStore) setFail(v bool) {
621	l.mu.Lock()
622	l.fail = v
623	l.mu.Unlock()
624	}
625
626	func (l *listFailingStore) failing() bool {
627	l.mu.Lock()
628	defer l.mu.Unlock()
629	return l.fail
630	}
631
632	func (l *listFailingStore) List(ctx context.Context, prefix string) ([]string, error) {
633	if l.failing() {
634	return nil, errors.New("bucket down")
635	}
636	return l.Store.List(ctx, prefix)
637	}
638
639	func (l *listFailingStore) Get(ctx context.Context, key string) ([]byte, string, error) {
640	if l.failing() {
641	return nil, "", errors.New("bucket down")
642	}
643	return l.Store.Get(ctx, key)
644	}
645
646	// Degrade, don't die: a failed poll keeps the last-good map. The alternative
647	// (an empty map) turns a bucket blip into a TLS outage on every non-issuer.
648	func TestCertMirrorRefreshErrorKeepsLastGood(t *testing.T) {
649	s := &listFailingStore{Store: release.NewDirStore(t.TempDir())}
650	putBucket(t, s.Store, "example.com", []byte("PEM"))
651	m := newCertMirror(s, nil)
652	if err := m.Refresh(context.Background()); err != nil {
653	t.Fatal(err)
654	}
655	s.setFail(true)
656	if err := m.Refresh(context.Background()); err == nil {
657	t.Fatal("a failed poll must be reported to the loop, which logs it")
658	}
659	if b, ok := m.Cert("example.com"); !ok || string(b) != "PEM" {
660	t.Fatalf("after a failed refresh the mirror = %q, %v — want the last-good copy", b, ok)
661	}
662	}
663
664	// A withdrawn entry (a spent challenge token the issuer deleted) leaves the
665	// map on the next good poll.
666	func TestCertMirrorRefreshDropsWithdrawnEntries(t *testing.T) {
667	s := release.NewDirStore(t.TempDir())
668	putBucket(t, s, "example.com", []byte("PEM"))
669	m := newCertMirror(s, nil)
670	if err := m.Refresh(context.Background()); err != nil {
671	t.Fatal(err)
672	}
673	if err := s.Delete(context.Background(), certKey("example.com")); err != nil {
674	t.Fatal(err)
675	}
676	if err := m.Refresh(context.Background()); err != nil {
677	t.Fatal(err)
678	}
679	if _, ok := m.Cert("example.com"); ok {
680	t.Fatal("a deleted bucket entry must leave the mirror")
681	}
682	}
683
684	// ---- R3: the mirror feeds this box's own autocert cache ---------------------
685
686	// localCert reads a name back out of a local autocert cache.
687	func localCert(t *testing.T, c autocert.Cache, name string) ([]byte, error) {
688	t.Helper()
689	return c.Get(context.Background(), name)
690	}
691
692	// autocert's renewal timers never consult HostPolicy, and renewal.go's do()
693	// checks the CACHE first — so a mirror-fed local cache is what stops a
694	// stood-down box's stale timers from ordering. It also carries a box through
695	// the boot window before the election has been decided.
696	func TestCertMirrorWritesFreshCertsIntoTheLocalCache(t *testing.T) {
697	const host = "app.bab.oncarlos.com"
698	s := release.NewDirStore(t.TempDir())
699	fresh := hostCertPEM(t, host, time.Now().Add(80*24*time.Hour))
700	putBucket(t, s, host, fresh)
701	putBucket(t, s, "wildcard-bab.oncarlos.com.pem", hostCertPEM(t, "*.bab.oncarlos.com", time.Now().Add(80*24*time.Hour)))
702
703	local := autocert.DirCache(t.TempDir())
704	m := newCertMirror(s, local)
705	if err := m.Refresh(context.Background()); err != nil {
706	t.Fatal(err)
707	}
708	got, err := localCert(t, local, host)
709	if err != nil || !bytes.Equal(got, fresh) {
710	t.Fatalf("local cache after refresh = %d byte(s), %v — want the mirrored cert", len(got), err)
711	}
712	// The wildcard PEM lands under wildcard.go's own cache name, which is what
713	// the wildcard manager's loadCached reads.
714	if _, err := localCert(t, local, "wildcard-bab.oncarlos.com.pem"); err != nil {
715	t.Fatalf("wildcard PEM did not reach the local cache: %v", err)
716	}
717	}
718
719	// A renewal in the bucket replaces an older local copy; a local copy that is
720	// already newer is left alone (this box may be the issuer, one poll ahead of
721	// what it published).
722	func TestCertMirrorLocalWriteRespectsFreshness(t *testing.T) {
723	const host = "app.bab.oncarlos.com"
724	s := release.NewDirStore(t.TempDir())
725	local := autocert.DirCache(t.TempDir())
726
727	old := hostCertPEM(t, host, time.Now().Add(10*24*time.Hour))
728	newer := hostCertPEM(t, host, time.Now().Add(80*24*time.Hour))
729
730	// Local holds the OLD one; the bucket holds a renewal.
731	if err := local.Put(context.Background(), host, old); err != nil {
732	t.Fatal(err)
733	}
734	putBucket(t, s, host, newer)
735	m := newCertMirror(s, local)
736	if err := m.Refresh(context.Background()); err != nil {
737	t.Fatal(err)
738	}
739	if got, _ := localCert(t, local, host); !bytes.Equal(got, newer) {
740	t.Fatal("a renewal in the bucket must replace the older local copy")
741	}
742
743	// Now the other way round: local is fresher than the bucket.
744	local2 := autocert.DirCache(t.TempDir())
745	if err := local2.Put(context.Background(), host, newer); err != nil {
746	t.Fatal(err)
747	}
748	s2 := release.NewDirStore(t.TempDir())
749	putBucket(t, s2, host, old)
750	m2 := newCertMirror(s2, local2)
751	if err := m2.Refresh(context.Background()); err != nil {
752	t.Fatal(err)
753	}
754	if got, _ := localCert(t, local2, host); !bytes.Equal(got, newer) {
755	t.Fatal("a local cert that is already fresher must not be downgraded by the mirror")
756	}
757	}
758
759	// Account keys reach the local cache for issuer-takeover warmth (one ACME
760	// account per deployment, one rate-limit bucket) — but only when this box has
761	// none: clobbering a registered account key is never an improvement.
762	func TestCertMirrorSeedsAccountKeysOnlyWhenAbsent(t *testing.T) {
763	s := release.NewDirStore(t.TempDir())
764	putBucket(t, s, "acme_account+key", []byte("BUCKET-KEY"))
765	putBucket(t, s, wildcardAccountKeyName, []byte("BUCKET-WILDCARD-KEY"))
766
767	local := autocert.DirCache(t.TempDir())
768	m := newCertMirror(s, local)
769	if err := m.Refresh(context.Background()); err != nil {
770	t.Fatal(err)
771	}
772	if got, err := localCert(t, local, "acme_account+key"); err != nil || string(got) != "BUCKET-KEY" {
773	t.Fatalf("account key not seeded: %q, %v", got, err)
774	}
775
776	local2 := autocert.DirCache(t.TempDir())
777	if err := local2.Put(context.Background(), "acme_account+key", []byte("MINE")); err != nil {
778	t.Fatal(err)
779	}
780	m2 := newCertMirror(s, local2)
781	if err := m2.Refresh(context.Background()); err != nil {
782	t.Fatal(err)
783	}
784	if got, _ := localCert(t, local2, "acme_account+key"); string(got) != "MINE" {
785	t.Fatalf("this box's own account key was clobbered: %q", got)
786	}
787	}
788
789	// Challenge tokens are ephemeral and the issuer owns their deletion: copying
790	// them onto every box's disk would litter caches nothing ever sweeps.
791	func TestCertMirrorNeverWritesChallengeTokensLocally(t *testing.T) {
792	const token = "0123456789abcdefghij0123456789abcdefghijkl"
793	s := release.NewDirStore(t.TempDir())
794	putBucket(t, s, token+"+http-01", []byte("resp"))
795	local := autocert.DirCache(t.TempDir())
796	m := newCertMirror(s, local)
797	if err := m.Refresh(context.Background()); err != nil {
798	t.Fatal(err)
799	}
800	if _, err := localCert(t, local, token+"+http-01"); !errors.Is(err, autocert.ErrCacheMiss) {
801	t.Fatalf("a challenge token must not land in the local cache: %v", err)
802	}
803	}
804
805	// A steady state costs no local I/O: an unchanged entry is not rewritten.
806	func TestCertMirrorSkipsUnchangedLocalWrites(t *testing.T) {
807	const host = "app.bab.oncarlos.com"
808	s := release.NewDirStore(t.TempDir())
809	putBucket(t, s, host, hostCertPEM(t, host, time.Now().Add(80*24*time.Hour)))
810	local := &countingWriteCache{Cache: autocert.DirCache(t.TempDir())}
811	m := newCertMirror(s, local)
812	for range 3 {
813	if err := m.Refresh(context.Background()); err != nil {
814	t.Fatal(err)
815	}
816	}
817	if n := local.puts(); n != 1 {
818	t.Fatalf("local writes = %d, want exactly 1 — an unchanged mirror must not rewrite the cache", n)
819	}
820	}
821
822	// countingWriteCache counts Puts.
823	type countingWriteCache struct {
824	autocert.Cache
825	mu sync.Mutex
826	n int
827	}
828
829	func (c *countingWriteCache) Put(ctx context.Context, key string, data []byte) error {
830	c.mu.Lock()
831	c.n++
832	c.mu.Unlock()
833	return c.Cache.Put(ctx, key, data)
834	}
835
836	func (c *countingWriteCache) puts() int {
837	c.mu.Lock()
838	defer c.mu.Unlock()
839	return c.n
840	}
841
842	// ---- R4: a wildcard takeover propagates on the poll -------------------------
843
844	func TestCertMirrorWakesTheWildcardManagerOnChangedWildcardMaterial(t *testing.T) {
845	s := release.NewDirStore(t.TempDir())
846	putBucket(t, s, "app.bab.oncarlos.com", hostCertPEM(t, "app.bab.oncarlos.com", time.Now().Add(80*24*time.Hour)))
847	local := autocert.DirCache(t.TempDir())
848	m := newCertMirror(s, local)
849	var wakes int
850	m.wake = func() { wakes++ }
851
852	if err := m.Refresh(context.Background()); err != nil {
853	t.Fatal(err)
854	}
855	if wakes != 0 {
856	t.Fatalf("a per-host cert is not wildcard material: %d wake(s)", wakes)
857	}
858
859	putBucket(t, s, "wildcard-bab.oncarlos.com.pem", hostCertPEM(t, "*.bab.oncarlos.com", time.Now().Add(80*24*time.Hour)))
860	if err := m.Refresh(context.Background()); err != nil {
861	t.Fatal(err)
862	}
863	if wakes != 1 {
864	t.Fatalf("a new wildcard must wake the manager: %d wake(s)", wakes)
865	}
866	if err := m.Refresh(context.Background()); err != nil {
867	t.Fatal(err)
868	}
869	if wakes != 1 {
870	t.Fatalf("an unchanged poll must not wake the manager: %d wake(s)", wakes)
871	}
872	putBucket(t, s, "wildcard-bab.oncarlos.com.pem", hostCertPEM(t, "*.bab.oncarlos.com", time.Now().Add(85*24*time.Hour)))
873	if err := m.Refresh(context.Background()); err != nil {
874	t.Fatal(err)
875	}
876	if wakes != 2 {
877	t.Fatalf("a renewed wildcard must wake the manager: %d wake(s)", wakes)
878	}
879	}
880
881	// ---- R1: the serving chain --------------------------------------------------
882
883	const mirrorHost = "app.bab.oncarlos.com"
884
885	// gatedChain builds what RunWithActivatorReady builds for a NON-ISSUER: a
886	// gated autocert manager with m in front of it and autocert's own cache
887	// (counted, so "did autocert serve this?" is observable) behind it.
888	//
889	// The mirror is passed in rather than built here so that a test can give it
890	// NO local cache: the mirror layer and the last-resort local-cache layer both
891	// serve the same bytes, and a test that means to pin one must not be able to
892	// pass on the other.
893	func gatedChain(t *testing.T, m *certMirror, local autocert.Cache, hosts ...string) (func(*tls.ClientHelloInfo) (*tls.Certificate, error), *countingCache) {
894	t.Helper()
895	return gatedChainFor(t, testIssuerEdge(hosts...), m, local)
896	}
897
898	// gatedChainFor is gatedChain over an edge the caller built — the form a test
899	// needs when the edge carries something beyond its route table, such as the
900	// mirrored home map the cert gates now consult (deploymentHost, #277).
901	func gatedChainFor(t *testing.T, e *Edge, m *certMirror, local autocert.Cache) (func(*tls.ClientHelloInfo) (*tls.Certificate, error), *countingCache) {
902	t.Helper()
903	e.issuer = func() (context.Context, bool) { return nil, false } // never the issuer
904	counting := &countingCache{Cache: local, gets: map[string]int{}}
905	mgr := offlineManager(t, counting, e.issuerHostPolicy())
906	return m.wrapGetCertificate(mgr.GetCertificate), counting
907	}
908
909	// The whole point of Task 5: a gated box serves the issuer's cert, and does
910	// so BEFORE autocert — which would refuse the name outright.
911	func TestMirrorServesTheGatedNonIssuersHandshake(t *testing.T) {
912	s := release.NewDirStore(t.TempDir())
913	putBucket(t, s, mirrorHost, hostCertPEM(t, mirrorHost, time.Now().Add(80*24*time.Hour)))
914	// No local cache on the mirror: only the mirror layer itself can answer.
915	m := newCertMirror(s, nil)
916	get, counting := gatedChain(t, m, autocert.DirCache(t.TempDir()), mirrorHost)
917	if err := m.Refresh(context.Background()); err != nil {
918	t.Fatal(err)
919	}
920	got, err := get(ecdsaHello(mirrorHost))
921	if err != nil {
922	t.Fatalf("a non-issuer must serve the mirrored cert: %v", err)
923	}
924	if got.Leaf == nil || got.Leaf.VerifyHostname(mirrorHost) != nil {
925	t.Fatalf("served cert does not cover %s", mirrorHost)
926	}
927	if n := counting.count(mirrorHost); n != 0 {
928	t.Fatalf("autocert was consulted %d time(s) — a valid mirror cert must short-circuit before it", n)
929	}
930	}
931
932	// Each layer's miss falls through cleanly. With no gate (a single-box
933	// deployment) and an empty mirror, autocert serves from its own cache exactly
934	// as it does today.
935	func TestMirrorFallsThroughToAutocert(t *testing.T) {
936	local := autocert.DirCache(t.TempDir())
937	if err := local.Put(context.Background(), mirrorHost, hostCertPEM(t, mirrorHost, time.Now().Add(80*24*time.Hour))); err != nil {
938	t.Fatal(err)
939	}
940	e := testIssuerEdge(mirrorHost) // ungated
941	m := newCertMirror(release.NewDirStore(t.TempDir()), local)
942	get := m.wrapGetCertificate(offlineManager(t, local, e.issuerHostPolicy()).GetCertificate)
943	if _, err := get(ecdsaHello(mirrorHost)); err != nil {
944	t.Fatalf("an empty mirror must fall through to autocert: %v", err)
945	}
946	}
947
948	// Last resort: gated autocert AND an empty mirror (a bucket outage across a
949	// restart), but the cert is on this box's disk from before. autocert refuses
950	// it — HostPolicy runs before its cache lookup — so the chain reads the
951	// DirCache directly rather than dropping the handshake.
952	func TestMirrorLastResortServesTheLocalCacheEvenWhenGated(t *testing.T) {
953	local := autocert.DirCache(t.TempDir())
954	if err := local.Put(context.Background(), mirrorHost, hostCertPEM(t, mirrorHost, time.Now().Add(80*24*time.Hour))); err != nil {
955	t.Fatal(err)
956	}
957	m := newCertMirror(release.NewDirStore(t.TempDir()), local)
958	get, _ := gatedChain(t, m, local, mirrorHost)
959	got, err := get(ecdsaHello(mirrorHost))
960	if err != nil {
961	t.Fatalf("the local cache is the last resort, and it has the cert: %v", err)
962	}
963	if got.Leaf.VerifyHostname(mirrorHost) != nil {
964	t.Fatal("served cert does not cover the host")
965	}
966	}
967
968	// Nothing anywhere: the error the caller sees is autocert's own, so the
969	// journal still says WHY (not-issuer, or unknown host) rather than blaming
970	// the mirror.
971	func TestMirrorChainKeepsAutocertsError(t *testing.T) {
972	get, _ := gatedChain(t, newCertMirror(release.NewDirStore(t.TempDir()), nil), autocert.DirCache(t.TempDir()), mirrorHost)
973	_, err := get(ecdsaHello(mirrorHost))
974	if !errors.Is(err, errNotIssuer) {
975	t.Fatalf("chain error = %v, want errNotIssuer", err)
976	}
977	}
978
979	// A mirrored entry that is expired, or does not cover the name asked for, is
980	// not served: the chain falls through instead. Serving a dead cert is worse
981	// than serving none — the handshake fails either way, but a fall-through can
982	// still find a good one.
983	func TestMirrorRefusesUnusableEntries(t *testing.T) {
984	for _, tc := range []struct {
985	name string
986	key string
987	pem []byte
988	}{
989	{"expired", mirrorHost, hostCertPEM(t, mirrorHost, time.Now().Add(-time.Hour))},
990	{"wrong host", mirrorHost, hostCertPEM(t, "other.bab.oncarlos.com", time.Now().Add(80*24*time.Hour))},
991	} {
992	t.Run(tc.name, func(t *testing.T) {
993	s := release.NewDirStore(t.TempDir())
994	putBucket(t, s, tc.key, tc.pem)
995	m := newCertMirror(s, nil)
996	get, _ := gatedChain(t, m, autocert.DirCache(t.TempDir()), mirrorHost)
997	if err := m.Refresh(context.Background()); err != nil {
998	t.Fatal(err)
999	}
1000	if _, err := get(ecdsaHello(mirrorHost)); !errors.Is(err, errNotIssuer) {
1001	t.Fatalf("an unusable mirrored entry must fall through, got %v", err)
1002	}
1003	})
1004	}
1005	}
1006
1007	// autocert keys legacy clients' certs under "<host>+rsa", so the mirror must
1008	// look under the SAME name for the SAME client — with both entries present,
1009	// the choice is the one autocert would have made.
1010	func TestMirrorPicksTheEntryAutocertWouldHave(t *testing.T) {
1011	s := release.NewDirStore(t.TempDir())
1012	ecdsaPEM := hostCertPEM(t, mirrorHost, time.Now().Add(80*24*time.Hour))
1013	rsaPEM := hostCertPEM(t, mirrorHost, time.Now().Add(70*24*time.Hour))
1014	putBucket(t, s, mirrorHost, ecdsaPEM)
1015	putBucket(t, s, mirrorHost+"+rsa", rsaPEM)
1016	m := newCertMirror(s, nil)
1017	get, _ := gatedChain(t, m, autocert.DirCache(t.TempDir()), mirrorHost)
1018	if err := m.Refresh(context.Background()); err != nil {
1019	t.Fatal(err)
1020	}
1021	rsaOnly := &tls.ClientHelloInfo{
1022	ServerName: mirrorHost,
1023	SignatureSchemes: []tls.SignatureScheme{tls.PKCS1WithSHA256},
1024	CipherSuites: []uint16{tls.TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256},
1025	}
1026	for _, tc := range []struct {
1027	name string
1028	hello *tls.ClientHelloInfo
1029	want []byte
1030	}{
1031	{"ecdsa client", ecdsaHello(mirrorHost), ecdsaPEM},
1032	{"rsa-only client", rsaOnly, rsaPEM},
1033	} {
1034	got, err := get(tc.hello)
1035	if err != nil {
1036	t.Fatalf("%s: %v", tc.name, err)
1037	}
1038	want, err := parseCertEntry(tc.want)
1039	if err != nil {
1040	t.Fatal(err)
1041	}
1042	if !bytes.Equal(got.Certificate[0], want.Certificate[0]) {
1043	t.Fatalf("%s was served the other entry", tc.name)
1044	}
1045	}
1046	}
1047
1048	// ---- tls-alpn-01: the challenge autocert PREFERS ----------------------------
1049
1050	func alpnHello(host string) *tls.ClientHelloInfo {
1051	h := ecdsaHello(host)
1052	h.SupportedProtos = []string{acme.ALPNProto}
1053	return h
1054	}
1055
1056	// autocert tries tls-alpn-01 first and only falls back to http-01 after a
1057	// whole failed order, so an N-edge deployment would burn a failed validation
1058	// on (N-1)/N of its orders if only the issuer could answer these.
1059	func TestMirrorAnswersAnotherEdgesALPNValidation(t *testing.T) {
1060	s := release.NewDirStore(t.TempDir())
1061	putBucket(t, s, mirrorHost+"+token", hostCertPEM(t, mirrorHost, time.Now().Add(24*time.Hour)))
1062	m := newCertMirror(s, nil)
1063	get, _ := gatedChain(t, m, autocert.DirCache(t.TempDir()), mirrorHost)
1064	m.allowed = func(h string) bool { return h == mirrorHost }
1065	// Deliberately NOT refreshed: a validation is live, and the poll is 30s.
1066	if _, err := get(alpnHello(mirrorHost)); err != nil {
1067	t.Fatalf("a non-issuer must answer another edge's tls-alpn-01 validation: %v", err)
1068	}
1069	}
1070
1071	// The SNI here is attacker-chosen, so the live lookup is bounded to the names
1072	// this DEPLOYMENT serves — this box's own routes, or another box's
1073	// exactly-named ones (#277). A stranger is refused before any bucket read, as
1074	// it always was.
1075	func TestMirrorALPNReadThroughOnlyForServedOrRemoteHomedHosts(t *testing.T) {
1076	s := &bucketGetCounter{Store: release.NewDirStore(t.TempDir())}
1077	m := newCertMirror(s, nil)
1078	get, _ := gatedChain(t, m, autocert.DirCache(t.TempDir()), mirrorHost)
1079	m.allowed = func(string) bool { return false }
1080	if _, err := get(alpnHello("stranger.example")); err == nil {
1081	t.Fatal("there is no token cert for a host this deployment does not serve")
1082	}
1083	if n := s.gets(); n != 0 {
1084	t.Fatalf("bucket gets = %d, want 0 for an unserved host", n)
1085	}
1086	}
1087
1088	// A token cert is NOT a certificate for the host: it is self-signed and only
1089	// meaningful to a validating CA. It must never leak into an ordinary
1090	// handshake.
1091	func TestMirrorNeverServesATokenCertToAnOrdinaryHello(t *testing.T) {
1092	s := release.NewDirStore(t.TempDir())
1093	putBucket(t, s, mirrorHost+"+token", hostCertPEM(t, mirrorHost, time.Now().Add(24*time.Hour)))
1094	m := newCertMirror(s, nil)
1095	get, _ := gatedChain(t, m, autocert.DirCache(t.TempDir()), mirrorHost)
1096	m.allowed = func(string) bool { return true }
1097	if err := m.Refresh(context.Background()); err != nil {
1098	t.Fatal(err)
1099	}
1100	if _, err := get(ecdsaHello(mirrorHost)); !errors.Is(err, errNotIssuer) {
1101	t.Fatalf("an ordinary hello must not be served the tls-alpn-01 token cert: %v", err)
1102	}
1103	}
1104
1105	// The wildcard stays in front of everything: a covered name never reaches the
1106	// mirror at all (this is the composition RunWithActivatorReady builds).
1107	func TestWildcardWinsAheadOfTheMirror(t *testing.T) {
1108	const parent = "bab.oncarlos.com"
1109	s := release.NewDirStore(t.TempDir())
1110	putBucket(t, s, mirrorHost, hostCertPEM(t, mirrorHost, time.Now().Add(80*24*time.Hour)))
1111	m := newCertMirror(s, nil)
1112	inner, _ := gatedChain(t, m, autocert.DirCache(t.TempDir()), mirrorHost)
1113	if err := m.Refresh(context.Background()); err != nil {
1114	t.Fatal(err)
1115	}
1116	w := newWildcardManager([]string{parent}, "", func() []string { return nil }, nil, "", autocert.DirCache(t.TempDir()), &fakeDNS{})
1117	wildcard, _ := selfSignedWildcard(t, parent, time.Now().Add(80*24*time.Hour))
1118	w.refresh()
1119	w.mu.Lock()
1120	w.certs[parent] = wildcard
1121	w.mu.Unlock()
1122
1123	got, err := wrapGetCertificate(w, inner)(ecdsaHello(mirrorHost))
1124	if err != nil {
1125	t.Fatal(err)
1126	}
1127	if got != wildcard {
1128	t.Fatal("a wildcard-covered name must be served the wildcard, not the mirror's per-host cert")
1129	}
1130	}
1131
1132	// ---- R2: challenges, in front of autocert's own handler ---------------------
1133
1134	const mirrorTestToken = "0123456789abcdefghij0123456789abcdefghijkl"
1135
1136	func challengeRequest(t *testing.T, h http.Handler, token string) *httptest.ResponseRecorder {
1137	t.Helper()
1138	return challengeRequestHost(t, h, mirrorHost, token)
1139	}
1140
1141	// gatedHTTPHandler is the :80 chain of a non-issuer: the mirror's handler in
1142	// front of autocert's, which 403s every challenge on this box.
1143	func gatedHTTPHandler(t *testing.T, m *certMirror) http.Handler {
1144	t.Helper()
1145	return gatedHTTPHandlerFor(t, testIssuerEdge(mirrorHost), m)
1146	}
1147
1148	// gatedHTTPHandlerFor is gatedHTTPHandler over an edge the caller built, for
1149	// the same reason gatedChainFor exists.
1150	func gatedHTTPHandlerFor(t *testing.T, e *Edge, m *certMirror) http.Handler {
1151	t.Helper()
1152	e.issuer = func() (context.Context, bool) { return nil, false }
1153	mgr := offlineManager(t, autocert.DirCache(t.TempDir()), e.issuerHostPolicy())
1154	fallback := http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
1155	w.Header().Set("X-Fallback", "yes")
1156	w.WriteHeader(http.StatusTeapot)
1157	})
1158	return m.challengeHandler(mgr.HTTPHandler(fallback))
1159	}
1160
1161	// autocert's HTTPHandler runs its HostPolicy — the issuance gate — before it
1162	// ever looks a token up, so a non-issuer 403s a challenge it is holding the
1163	// answer to. The mirror answers first.
1164	func TestChallengeHandlerAnswersInFrontOfAutocert(t *testing.T) {
1165	s := release.NewDirStore(t.TempDir())
1166	putBucket(t, s, mirrorTestToken+"+http-01", []byte("keyauth"))
1167	m := newCertMirror(s, nil)
1168	if err := m.Refresh(context.Background()); err != nil {
1169	t.Fatal(err)
1170	}
1171	got := challengeRequest(t, gatedHTTPHandler(t, m), mirrorTestToken)
1172	if got.Code != 200 || got.Body.String() != "keyauth" {
1173	t.Fatalf("challenge = %d %q, want 200 \"keyauth\"", got.Code, got.Body.String())
1174	}
1175	}
1176
1177	// Everything that is not a challenge path is somebody else's request.
1178	func TestChallengeHandlerDelegatesEverythingElse(t *testing.T) {
1179	m := newCertMirror(release.NewDirStore(t.TempDir()), nil)
1180	r := httptest.NewRequest("GET", "http://"+mirrorHost+"/whatever", nil)
1181	w := httptest.NewRecorder()
1182	gatedHTTPHandler(t, m).ServeHTTP(w, r)
1183	if w.Header().Get("X-Fallback") != "yes" {
1184	t.Fatalf("non-challenge request = %d, want the fallback handler", w.Code)
1185	}
1186	}
1187
1188	// A live validation is faster than a 30s poll: the token was written to the
1189	// bucket a second ago and Accept() has already been sent. A miss reads
1190	// through — once, then from a short-lived cache.
1191	func TestChallengeReadsThroughForLiveTokens(t *testing.T) {
1192	s := release.NewDirStore(t.TempDir())
1193	putBucket(t, s, mirrorTestToken+"+http-01", []byte("keyauth"))
1194	counting := &bucketGetCounter{Store: s}
1195	m := newCertMirror(counting, nil) // deliberately never refreshed
1196	h := gatedHTTPHandler(t, m)
1197	for range 3 {
1198	got := challengeRequest(t, h, mirrorTestToken)
1199	if got.Code != 200 || got.Body.String() != "keyauth" {
1200	t.Fatalf("read-through challenge = %d %q", got.Code, got.Body.String())
1201	}
1202	}
1203	if n := counting.gets(); n != 1 {
1204	t.Fatalf("bucket gets = %d, want 1 — a live token is cached briefly after the first read", n)
1205	}
1206	}
1207
1208	// A miss is negative-cached: a scanner walking /.well-known/acme-challenge/
1209	// must not turn into one bucket GET per request.
1210	func TestChallengeReadThroughNegativeCachesMisses(t *testing.T) {
1211	counting := &bucketGetCounter{Store: release.NewDirStore(t.TempDir())}
1212	m := newCertMirror(counting, nil)
1213	h := gatedHTTPHandler(t, m)
1214	for range 5 {
1215	if got := challengeRequest(t, h, mirrorTestToken); got.Code == 200 {
1216	t.Fatal("there is no such token; the handler must not answer 200")
1217	}
1218	}
1219	if n := counting.gets(); n != 1 {
1220	t.Fatalf("bucket gets = %d, want 1 — misses are negative-cached", n)
1221	}
1222	}
1223
1224	// Only names shaped like an ACME token are ever fetched: the read-through is
1225	// a bucket read driven by an unauthenticated URL path.
1226	func TestChallengeReadThroughRefusesNonTokenPaths(t *testing.T) {
1227	counting := &bucketGetCounter{Store: release.NewDirStore(t.TempDir())}
1228	m := newCertMirror(counting, nil)
1229	h := gatedHTTPHandler(t, m)
1230	for _, token := range []string{"short", "../../control/serial", "wildcard-account.key", strings.Repeat("a", 300)} {
1231	if got := challengeRequest(t, h, token); got.Code == 200 {
1232	t.Fatalf("%q answered 200", token)
1233	}
1234	}
1235	if n := counting.gets(); n != 0 {
1236	t.Fatalf("bucket gets = %d, want 0 — a name that cannot be an ACME token is never fetched", n)
1237	}
1238	}
1239
1240	// Distinct token names defeat the negative cache, so the read-through carries
1241	// its own budget: a flood degrades to "no read-through", never to a bucket
1242	// GET per request.
1243	func TestChallengeReadThroughIsRateLimited(t *testing.T) {
1244	counting := &bucketGetCounter{Store: release.NewDirStore(t.TempDir())}
1245	m := newCertMirror(counting, nil)
1246	h := gatedHTTPHandler(t, m)
1247	for i := range 200 {
1248	challengeRequest(t, h, fmt.Sprintf("%s%03d", strings.Repeat("z", 39), i))
1249	}
1250	if n := counting.gets(); n > readThroughBurst {
1251	t.Fatalf("bucket gets = %d under a flood, want <= %d", n, readThroughBurst)
1252	}
1253	if counting.gets() == 0 {
1254	t.Fatal("test bug: the read-through never ran at all")
1255	}
1256	}
1257
1258	// bucketGetCounter counts Gets.
1259	type bucketGetCounter struct {
1260	release.Store
1261	mu sync.Mutex
1262	n int
1263	}
1264
1265	func (c *bucketGetCounter) Get(ctx context.Context, key string) ([]byte, string, error) {
1266	c.mu.Lock()
1267	c.n++
1268	c.mu.Unlock()
1269	return c.Store.Get(ctx, key)
1270	}
1271
1272	func (c *bucketGetCounter) gets() int {
1273	c.mu.Lock()
1274	defer c.mu.Unlock()
1275	return c.n
1276	}
1277
1278	// slowGetStore blocks every Get until release is closed.
1279	type slowGetStore struct {
1280	release.Store
1281	release chan struct{}
1282	mu sync.Mutex
1283	n int
1284	}
1285
1286	func (s *slowGetStore) Get(ctx context.Context, key string) ([]byte, string, error) {
1287	s.mu.Lock()
1288	s.n++
1289	s.mu.Unlock()
1290	<-s.release
1291	return s.Store.Get(ctx, key)
1292	}
1293
1294	func (s *slowGetStore) gets() int {
1295	s.mu.Lock()
1296	defer s.mu.Unlock()
1297	return s.n
1298	}
1299
1300	// A CA opens several connections for one validation. They share one bucket
1301	// read: the slow one is in flight, the rest wait on it.
1302	func TestChallengeReadThroughIsSingleFlighted(t *testing.T) {
1303	inner := release.NewDirStore(t.TempDir())
1304	putBucket(t, inner, mirrorTestToken+"+http-01", []byte("keyauth"))
1305	s := &slowGetStore{Store: inner, release: make(chan struct{})}
1306	m := newCertMirror(s, nil)
1307	h := gatedHTTPHandler(t, m)
1308
1309	var wg sync.WaitGroup
1310	got := make([]int, 6)
1311	for i := range got {
1312	wg.Add(1)
1313	go func() {
1314	defer wg.Done()
1315	got[i] = challengeRequest(t, h, mirrorTestToken).Code
1316	}()
1317	}
1318	// Let the waiters pile up behind the leader, then let it through.
1319	time.Sleep(50 * time.Millisecond)
1320	close(s.release)
1321	wg.Wait()
1322
1323	for i, code := range got {
1324	if code != 200 {
1325	t.Fatalf("request %d = %d, want 200", i, code)
1326	}
1327	}
1328	if n := s.gets(); n != 1 {
1329	t.Fatalf("bucket gets = %d, want 1 — concurrent lookups share one read", n)
1330	}
1331	}
1332
1333	// The loop is the whole delivery mechanism: it must pick material up without
1334	// anybody asking, keep going after a failure, and stop when told.
1335	func TestCertMirrorLoopPollsAndStops(t *testing.T) {
1336	prev := certMirrorInterval
1337	certMirrorInterval = 5 * time.Millisecond
1338	t.Cleanup(func() { certMirrorInterval = prev })
1339	log.SetOutput(io.Discard)
1340	t.Cleanup(func() { log.SetOutput(os.Stderr) })
1341
1342	s := &listFailingStore{Store: release.NewDirStore(t.TempDir())}
1343	s.setFail(true) // the loop starts against a broken bucket
1344	m := newCertMirror(s, nil)
1345	stop := make(chan struct{})
1346	done := make(chan struct{})
1347	go func() {
1348	m.run(stop)
1349	close(done)
1350	}()
1351
1352	putBucket(t, s.Store, "example.com", []byte("PEM"))
1353	s.setFail(false)
1354	deadline := time.Now().Add(2 * time.Second)
1355	for {
1356	if _, ok := m.Cert("example.com"); ok {
1357	break
1358	}
1359	if time.Now().After(deadline) {
1360	t.Fatal("the loop never picked the cert up")
1361	}
1362	time.Sleep(2 * time.Millisecond)
1363	}
1364	close(stop)
1365	select {
1366	case <-done:
1367	case <-time.After(2 * time.Second):
1368	t.Fatal("the loop did not stop")
1369	}
1370	}
1371
1372	// autocert arms its renewal timers LAZILY, from GetCertificate: m.cert()
1373	// loads the cache and calls startRenew (autocert.go:464), and nothing else
1374	// ever does. A mirror that short-circuited every hello would therefore
1375	// silently stop an ISSUER from renewing anything it had not ordered in this
1376	// process — the certs would simply expire in 90 days.
1377	//
1378	// So the short-circuit is for the GATED case only, which is the case it
1379	// exists for: an ungated box's autocert can serve everything the mirror holds
1380	// anyway, because syncLocal has already put it in its cache.
1381	func TestMirrorLetsAnIssuersAutocertSeeTheHello(t *testing.T) {
1382	s := release.NewDirStore(t.TempDir())
1383	putBucket(t, s, mirrorHost, hostCertPEM(t, mirrorHost, time.Now().Add(80*24*time.Hour)))
1384
1385	local := autocert.DirCache(t.TempDir())
1386	counting := &countingCache{Cache: local, gets: map[string]int{}}
1387	m := newCertMirror(s, local)
1388	m.gated = func() bool { return false } // this box IS the issuer
1389	if err := m.Refresh(context.Background()); err != nil {
1390	t.Fatal(err)
1391	}
1392	e := testIssuerEdge(mirrorHost) // ungated host policy
1393	get := m.wrapGetCertificate(offlineManager(t, counting, e.issuerHostPolicy()).GetCertificate)
1394	if _, err := get(ecdsaHello(mirrorHost)); err != nil {
1395	t.Fatalf("the issuer must serve the mirrored cert (through autocert): %v", err)
1396	}
1397	if n := counting.count(mirrorHost); n == 0 {
1398	t.Fatal("autocert never saw the hello — its renewal timer for this host is never armed")
1399	}
1400	}
1401
1402	// The mirror is still there behind autocert on an ungated box: a name
1403	// autocert cannot serve (and, being offline, cannot order) is rescued.
1404	func TestMirrorStillRescuesAnUngatedBox(t *testing.T) {
1405	s := release.NewDirStore(t.TempDir())
1406	putBucket(t, s, mirrorHost, hostCertPEM(t, mirrorHost, time.Now().Add(80*24*time.Hour)))
1407	m := newCertMirror(s, nil)
1408	m.gated = func() bool { return false }
1409	if err := m.Refresh(context.Background()); err != nil {
1410	t.Fatal(err)
1411	}
1412	e := testIssuerEdge(mirrorHost)
1413	get := m.wrapGetCertificate(offlineManager(t, autocert.DirCache(t.TempDir()), e.issuerHostPolicy()).GetCertificate)
1414	if _, err := get(ecdsaHello(mirrorHost)); err != nil {
1415	t.Fatalf("an ungated box whose order fails must still fall back to the mirror: %v", err)
1416	}
1417	}
1418
1419	// A mirror with no gate wired (every test, and any future caller that
1420	// forgets) behaves as though this box were gated: it serves, which is always
1421	// safe, rather than assuming an issuance authority it was never given.
1422	func TestMirrorWithNoGateDefaultsToServingFirst(t *testing.T) {
1423	s := release.NewDirStore(t.TempDir())
1424	putBucket(t, s, mirrorHost, hostCertPEM(t, mirrorHost, time.Now().Add(80*24*time.Hour)))
1425	m := newCertMirror(s, nil) // no gate
1426	if err := m.Refresh(context.Background()); err != nil {
1427	t.Fatal(err)
1428	}
1429	local := autocert.DirCache(t.TempDir())
1430	if err := local.Put(context.Background(), mirrorHost, hostCertPEM(t, mirrorHost, time.Now().Add(80*24*time.Hour))); err != nil {
1431	t.Fatal(err)
1432	}
1433	counting := &countingCache{Cache: local, gets: map[string]int{}}
1434	e := testIssuerEdge(mirrorHost) // ungated host policy: autocert WOULD serve
1435	get := m.wrapGetCertificate(offlineManager(t, counting, e.issuerHostPolicy()).GetCertificate)
1436	if _, err := get(ecdsaHello(mirrorHost)); err != nil {
1437	t.Fatal(err)
1438	}
1439	if n := counting.count(mirrorHost); n != 0 {
1440	t.Fatalf("autocert was consulted %d time(s) — with no gate the mirror answers first", n)
1441	}
1442	}
1443
1444	// ---- review fixes -----------------------------------------------------------
1445
1446	// keyFailingStore fails Get for exactly the keys named (all of them when
1447	// `all` is set), and counts every Get.
1448	type keyFailingStore struct {
1449	release.Store
1450	mu sync.Mutex
1451	fail map[string]bool
1452	all bool
1453	n int
1454	}
1455
1456	func (k *keyFailingStore) Get(ctx context.Context, key string) ([]byte, string, error) {
1457	k.mu.Lock()
1458	k.n++
1459	bad := k.all || k.fail[key]
1460	k.mu.Unlock()
1461	if bad {
1462	return nil, "", errors.New("object unreadable")
1463	}
1464	return k.Store.Get(ctx, key)
1465	}
1466
1467	func (k *keyFailingStore) setFail(keys ...string) {
1468	k.mu.Lock()
1469	k.fail = map[string]bool{}
1470	for _, key := range keys {
1471	k.fail[key] = true
1472	}
1473	k.mu.Unlock()
1474	}
1475
1476	func (k *keyFailingStore) setAll(v bool) {
1477	k.mu.Lock()
1478	k.all = v
1479	k.mu.Unlock()
1480	}
1481
1482	func (k *keyFailingStore) gets() int {
1483	k.mu.Lock()
1484	defer k.mu.Unlock()
1485	return k.n
1486	}
1487
1488	// FINDING 3: one unreadable object used to discard the WHOLE rebuild — every
1489	// unrelated new cert with it, plus syncLocal and the wildcard wake. A poll
1490	// fails only when the listing fails, or when nothing at all could be read.
1491	func TestCertMirrorPollCarriesUnreadableObjectsForward(t *testing.T) {
1492	inner := release.NewDirStore(t.TempDir())
1493	s := &keyFailingStore{Store: inner}
1494	putBucket(t, inner, "a.example.com", []byte("A"))
1495	putBucket(t, inner, "b.example.com", []byte("B"))
1496	local := autocert.DirCache(t.TempDir())
1497	m := newCertMirror(s, local)
1498	if err := m.Refresh(context.Background()); err != nil {
1499	t.Fatal(err)
1500	}
1501
1502	// a.example.com goes unreadable; a brand-new cert lands at the same time.
1503	s.setFail(certKey("a.example.com"))
1504	putBucket(t, inner, "c.example.com", hostCertPEM(t, "c.example.com", time.Now().Add(80*24*time.Hour)))
1505	if err := m.Refresh(context.Background()); err != nil {
1506	t.Fatalf("one bad object must not fail the whole poll: %v", err)
1507	}
1508	if b, ok := m.Cert("a.example.com"); !ok || string(b) != "A" {
1509	t.Fatalf("the unreadable object's previous copy = %q, %v — want it carried forward", b, ok)
1510	}
1511	if _, ok := m.Cert("c.example.com"); !ok {
1512	t.Fatal("the new cert must land even though a sibling object was unreadable")
1513	}
1514	if _, err := local.Get(context.Background(), "c.example.com"); err != nil {
1515	t.Fatalf("syncLocal must still run on a partially failed poll: %v", err)
1516	}
1517
1518	// Everything unreadable IS a failed poll — and the last-good map stands.
1519	s.setAll(true)
1520	if err := m.Refresh(context.Background()); err == nil {
1521	t.Fatal("a poll where every object failed must be reported")
1522	}
1523	if _, ok := m.Cert("b.example.com"); !ok {
1524	t.Fatal("a failed poll must keep the last-good map")
1525	}
1526	}
1527
1528	// FINDING 1: tls-alpn-01 token certs are keyed by HOSTNAME, not by a unique
1529	// token, so a previous order's entry for the same host is a live collision.
1530	// The snapshot is up to a poll stale (and, if a gated Delete leaked, far
1531	// staler than that) — token certs are read live, never served from the poll.
1532	func TestALPNTokenCertIsReadLiveNotFromTheSnapshot(t *testing.T) {
1533	inner := release.NewDirStore(t.TempDir())
1534	s := &bucketGetCounter{Store: inner}
1535	stale := hostCertPEM(t, mirrorHost, time.Now().Add(24*time.Hour))
1536	putBucket(t, inner, mirrorHost+"+token", stale)
1537
1538	m := newCertMirror(s, nil)
1539	m.allowed = func(h string) bool { return h == mirrorHost }
1540	if err := m.Refresh(context.Background()); err != nil {
1541	t.Fatal(err)
1542	}
1543	// A new order for the same host replaces the token cert in the bucket.
1544	fresh := hostCertPEM(t, mirrorHost, time.Now().Add(23*time.Hour))
1545	putBucket(t, inner, mirrorHost+"+token", fresh)
1546
1547	get, _ := gatedChain(t, m, autocert.DirCache(t.TempDir()), mirrorHost)
1548	got, err := get(alpnHello(mirrorHost))
1549	if err != nil {
1550	t.Fatal(err)
1551	}
1552	want, err := parseCertEntry(fresh)
1553	if err != nil {
1554	t.Fatal(err)
1555	}
1556	if !bytes.Equal(got.Certificate[0], want.Certificate[0]) {
1557	t.Fatal("the CA was handed the PREVIOUS order's token cert — the validation fails and the order burns")
1558	}
1559	if s.gets() == 0 {
1560	t.Fatal("a token cert must be read live, not served out of the poll's snapshot")
1561	}
1562	}
1563
1564	// ...and a token read-through is remembered for seconds, not for a poll: two
1565	// orders for one host can be minutes apart, but they can also be seconds.
1566	func TestALPNTokenReadThroughTTLIsShort(t *testing.T) {
1567	if readThroughTTL(mirrorHost+alpnTokenSuffix) > 5*time.Second {
1568	t.Fatalf("token TTL = %s, want a few seconds at most", readThroughTTL(mirrorHost+alpnTokenSuffix))
1569	}
1570	if got := readThroughTTL("tok" + httpTokenSuffix); got != readThroughHitTTL {
1571	t.Fatalf("HTTP-01 token TTL = %s, want %s — those names are unique per order", got, readThroughHitTTL)
1572	}
1573
1574	prev := readThroughTokenTTL
1575	readThroughTokenTTL = 0
1576	t.Cleanup(func() { readThroughTokenTTL = prev })
1577	inner := release.NewDirStore(t.TempDir())
1578	s := &bucketGetCounter{Store: inner}
1579	putBucket(t, inner, mirrorHost+"+token", hostCertPEM(t, mirrorHost, time.Now().Add(24*time.Hour)))
1580	m := newCertMirror(s, nil)
1581	m.allowed = func(string) bool { return true }
1582	get, _ := gatedChain(t, m, autocert.DirCache(t.TempDir()), mirrorHost)
1583	for range 2 {
1584	if _, err := get(alpnHello(mirrorHost)); err != nil {
1585	t.Fatal(err)
1586	}
1587	}
1588	if n := s.gets(); n != 2 {
1589	t.Fatalf("bucket gets = %d, want 2 — an expired token memo must be re-read", n)
1590	}
1591	}
1592
1593	// FINDING 2: some clients (curl, and anything not doing IDNA itself) send
1594	// unicode SNI. autocert converts with idna.Lookup.ToASCII before it looks in
1595	// its cache (autocert.go:307); a mirror that did not would hard-fail a
1596	// handshake the same box serves today.
1597	func TestMirrorConvertsUnicodeSNIToPunycode(t *testing.T) {
1598	const unicodeHost, asciiHost = "bücher.example", "xn--bcher-kva.example"
1599	s := release.NewDirStore(t.TempDir())
1600	putBucket(t, s, asciiHost, hostCertPEM(t, asciiHost, time.Now().Add(80*24*time.Hour)))
1601	m := newCertMirror(s, nil)
1602	if err := m.Refresh(context.Background()); err != nil {
1603	t.Fatal(err)
1604	}
1605	get, _ := gatedChain(t, m, autocert.DirCache(t.TempDir()), asciiHost)
1606	got, err := get(ecdsaHello(unicodeHost))
1607	if err != nil {
1608	t.Fatalf("unicode SNI must find the punycode cache name: %v", err)
1609	}
1610	if got.Leaf.VerifyHostname(asciiHost) != nil {
1611	t.Fatal("served cert does not cover the punycode host")
1612	}
1613	// The last-resort local read looks under the same converted name.
1614	local := autocert.DirCache(t.TempDir())
1615	if err := local.Put(context.Background(), asciiHost, hostCertPEM(t, asciiHost, time.Now().Add(80*24*time.Hour))); err != nil {
1616	t.Fatal(err)
1617	}
1618	m2 := newCertMirror(release.NewDirStore(t.TempDir()), local)
1619	get2, _ := gatedChain(t, m2, local, asciiHost)
1620	if _, err := get2(ecdsaHello(unicodeHost)); err != nil {
1621	t.Fatalf("the local last resort must convert too: %v", err)
1622	}
1623	}
1624
1625	// A name IDNA cannot convert (and an empty one) must not panic or fetch: it
1626	// falls through as an ordinary miss.
1627	func TestMirrorSurvivesUnconvertibleSNI(t *testing.T) {
1628	m := newCertMirror(release.NewDirStore(t.TempDir()), nil)
1629	get, _ := gatedChain(t, m, autocert.DirCache(t.TempDir()), mirrorHost)
1630	for _, sni := range []string{"", "-.-", strings.Repeat("x", 300) + ".example"} {
1631	if _, err := get(ecdsaHello(sni)); err == nil {
1632	t.Fatalf("%q: want a miss, got a certificate", sni)
1633	}
1634	}
1635	}
1636
1637	// challengeRequestHost is challengeRequest with an explicit Host header.
1638	func challengeRequestHost(t *testing.T, h http.Handler, host, token string) *httptest.ResponseRecorder {
1639	t.Helper()
1640	r := httptest.NewRequest("GET", "http://"+host+"/.well-known/acme-challenge/"+token, nil)
1641	w := httptest.NewRecorder()
1642	h.ServeHTTP(w, r)
1643	return w
1644	}
1645
1646	// FINDING 4a: the live lookup on :80 is bounded by the host under
1647	// validation, exactly as autocert's own handler bounds its token lookup. A
1648	// real CA always sends it; a flood against hosts this box does not serve
1649	// never reaches the bucket.
1650	func TestChallengeReadThroughIsBoundedByHost(t *testing.T) {
1651	inner := release.NewDirStore(t.TempDir())
1652	putBucket(t, inner, mirrorTestToken+"+http-01", []byte("keyauth"))
1653	s := &bucketGetCounter{Store: inner}
1654	m := newCertMirror(s, nil)
1655	m.allowed = func(h string) bool { return h == mirrorHost }
1656	h := gatedHTTPHandler(t, m)
1657
1658	if got := challengeRequestHost(t, h, "stranger.example", mirrorTestToken); got.Code == 200 {
1659	t.Fatal("a host this box does not serve must not drive a bucket read")
1660	}
1661	if n := s.gets(); n != 0 {
1662	t.Fatalf("bucket gets for an unserved host = %d, want 0", n)
1663	}
1664	if got := challengeRequestHost(t, h, mirrorHost+":80", mirrorTestToken); got.Code != 200 {
1665	t.Fatalf("the host under validation must be answered: %d", got.Code)
1666	}
1667	}
1668
1669	// FINDING 4b: :80 and 443 do not share one budget. An unauthenticated flood
1670	// of challenge requests must not starve a tls-alpn-01 rescue on 443.
1671	func TestChallengeFloodDoesNotStarveALPNRescue(t *testing.T) {
1672	inner := release.NewDirStore(t.TempDir())
1673	putBucket(t, inner, mirrorHost+"+token", hostCertPEM(t, mirrorHost, time.Now().Add(24*time.Hour)))
1674	s := &bucketGetCounter{Store: inner}
1675	m := newCertMirror(s, nil)
1676	m.allowed = func(string) bool { return true }
1677	h := gatedHTTPHandler(t, m)
1678	for i := range 200 {
1679	challengeRequest(t, h, fmt.Sprintf("%s%03d", strings.Repeat("z", 39), i))
1680	}
1681	get, _ := gatedChain(t, m, autocert.DirCache(t.TempDir()), mirrorHost)
1682	if _, err := get(alpnHello(mirrorHost)); err != nil {
1683	t.Fatalf("a :80 flood starved the 443 validation path: %v", err)
1684	}
1685	}
1686
1687	// ---- #277: the three gates admit the deployment's exactly-named hosts -------
1688	//
1689	// DNS for a home-boxed route can point at ANY edge. The edge it points at
1690	// backhauls to the home box with the real SNI, the home box (as issuer)
1691	// orders — and the CA then validates against the box DNS names, which is the
1692	// one whose registry does not carry the host. Bounding the gates by Allowed
1693	// meant every such validation failed, five failures burned the name's Let's
1694	// Encrypt per-hostname budget in minutes, and the route was down throughout.
1695	//
1696	// The gates now ask deploymentHost: this box's routes, OR an exact key of the
1697	// mirrored home map. Exact is the whole discipline — every test below that
1698	// says "wildcard label" is PR #180's rule standing on the new surface.
1699
1700	// remoteHomedEdge is one box of a two-box deployment: it serves served from
1701	// its own registry, and mirrors an edges file in which box "betty" homes
1702	// homed (pass "*.<parent>" to mirror a catch-all row, which Task 1 publishes
1703	// verbatim). The handle it issues and mirrors through COUNTS bucket Gets, so
1704	// "did this gate spend a read-through?" is observable; the home map polls the
1705	// same directory through its own handle, so the map's own reads never show up
1706	// in that count.
1707	func remoteHomedEdge(t *testing.T, served []string, homed ...string) (*Edge, *bucketGetCounter, release.Store) {
1708	t.Helper()
1709	base := release.NewDirStore(t.TempDir())
1710	counting := &bucketGetCounter{Store: base}
1711	e := testIssuerEdge(served...)
1712	e.staticStore = counting
1713	m := newHomeMap(base, "annie")
1714	homedOn(t, m, "betty", "2.2.2.2", homed...)
1715	e.homes.Store(m)
1716	return e, counting, base
1717	}
1718
1719	const (
1720	// remoteHomedHost is homed on the OTHER box: absent from this box's
1721	// registry, exactly named in its home map.
1722	remoteHomedHost = "app.remote.test"
1723	// remoteCatchAll is the catch-all row that box publishes, and
1724	// remoteWildcardLabel is a label under it — dialable through Home()'s
1725	// one-label rule, and admissible by none of the three cert gates.
1726	remoteCatchAll = "*.remote.test"
1727	remoteWildcardLabel = "invented.remote.test"
1728	)
1729
1730	// HTTP-01, the failing half of #277: the CA arrives at a box that does not
1731	// serve the name, and the token is in the bucket. It is answered.
1732	func TestChallengeReadThroughAdmitsRemoteHomedHost(t *testing.T) {
1733	e, counting, base := remoteHomedEdge(t, []string{mirrorHost}, remoteHomedHost)
1734	putBucket(t, base, mirrorTestToken+"+http-01", []byte("keyauth"))
1735	m := e.newCertMirror(nil) // production wiring: m.allowed = e.deploymentHost
1736	if m == nil {
1737	t.Fatal("a store-backed edge must have a mirror")
1738	}
1739	if e.Allowed(remoteHomedHost) {
1740	t.Fatal("test bug: the host is supposed to be absent from THIS box's registry")
1741	}
1742	got := challengeRequestHost(t, gatedHTTPHandlerFor(t, e, m), remoteHomedHost, mirrorTestToken)
1743	if got.Code != 200 || got.Body.String() != "keyauth" {
1744	t.Fatalf("challenge for a remote-homed host = %d %q, want 200 \"keyauth\"", got.Code, got.Body.String())
1745	}
1746	if n := counting.gets(); n != 1 {
1747	t.Fatalf("bucket gets = %d, want exactly 1 — the live read-through", n)
1748	}
1749	}
1750
1751	// ...and the anti-stranger bound the widening had to survive: a name in
1752	// neither the registry nor the map spends nothing at all, and neither does an
1753	// invented label under a MIRRORED catch-all (#180).
1754	func TestChallengeReadThroughStillRefusesStrangers(t *testing.T) {
1755	for _, host := range []string{"stranger.example", remoteWildcardLabel, remoteCatchAll} {
1756	t.Run(host, func(t *testing.T) {
1757	e, counting, base := remoteHomedEdge(t, []string{mirrorHost}, remoteHomedHost, remoteCatchAll)
1758	putBucket(t, base, mirrorTestToken+"+http-01", []byte("keyauth"))
1759	m := e.newCertMirror(nil)
1760	if got := challengeRequestHost(t, gatedHTTPHandlerFor(t, e, m), host, mirrorTestToken); got.Code == 200 {
1761	t.Fatalf("%s was answered a live token body", host)
1762	}
1763	if n := counting.gets(); n != 0 {
1764	t.Fatalf("bucket gets = %d, want 0 — %s must never drive a bucket read", n, host)
1765	}
1766	})
1767	}
1768	}
1769
1770	// tls-alpn-01, the half autocert tries FIRST: the validation cert for another
1771	// edge's order is served for a remote-homed name.
1772	func TestMirrorALPNReadThroughAdmitsRemoteHomedHost(t *testing.T) {
1773	e, _, base := remoteHomedEdge(t, []string{mirrorHost}, remoteHomedHost)
1774	putBucket(t, base, remoteHomedHost+"+token", hostCertPEM(t, remoteHomedHost, time.Now().Add(24*time.Hour)))
1775	m := e.newCertMirror(nil)
1776	get, _ := gatedChainFor(t, e, m, autocert.DirCache(t.TempDir()))
1777	// Deliberately NOT refreshed: a validation is live, and the poll is 30s.
1778	if _, err := get(alpnHello(remoteHomedHost)); err != nil {
1779	t.Fatalf("an edge DNS points at must answer the home box's tls-alpn-01 validation: %v", err)
1780	}
1781	}
1782
1783	// The #180 pin on the 443 surface: SNI is attacker-chosen, and a mirrored
1784	// catch-all must not turn an invented label into a bucket read.
1785	func TestALPNReadThroughRefusesWildcardLabelUnderRemoteCatchAll(t *testing.T) {
1786	e, counting, base := remoteHomedEdge(t, []string{mirrorHost}, remoteCatchAll)
1787	putBucket(t, base, remoteWildcardLabel+"+token", hostCertPEM(t, remoteWildcardLabel, time.Now().Add(24*time.Hour)))
1788	if _, _, ok := e.homes.Load().Home(remoteWildcardLabel); !ok {
1789	t.Fatal("test bug: the label is supposed to be DIALABLE through the one-label rule")
1790	}
1791	m := e.newCertMirror(nil)
1792	get, _ := gatedChainFor(t, e, m, autocert.DirCache(t.TempDir()))
1793	if _, err := get(alpnHello(remoteWildcardLabel)); err == nil {
1794	t.Fatal("a label under a mirrored catch-all must not be served a token cert")
1795	}
1796	if n := counting.gets(); n != 0 {
1797	t.Fatalf("bucket gets = %d, want 0 — the label must not spend the ALPN budget", n)
1798	}
1799	}
1800
1801	// The order gate: the issuer mints for any exactly-named route in the
1802	// deployment, whichever box homes it. Without this the home-boxed route can
1803	// only ever mint while the housekeeper lease sits on its own box.
1804	func TestIssuerOrdersForRemoteHomedHost(t *testing.T) {
1805	e, _, _ := remoteHomedEdge(t, []string{mirrorHost}, remoteHomedHost)
1806	e.issuer = func() (context.Context, bool) { return context.Background(), true }
1807	if err := e.issuerHostPolicy()(context.Background(), remoteHomedHost); err != nil {
1808	t.Fatalf("HostPolicy for a remote-homed host = %v, want nil", err)
1809	}
1810	if err := e.issuerHostPolicy()(context.Background(), mirrorHost); err != nil {
1811	t.Fatalf("HostPolicy for this box's own route = %v, want nil", err)
1812	}
1813	}
1814
1815	// The #180 pin on the order gate, and the one that matters most: an
1816	// attacker-chosen label must never drive a fresh per-host ACME order against
1817	// the deployment's single shared account and rate-limit bucket.
1818	func TestHostPolicyRefusesWildcardLabelUnderRemoteCatchAll(t *testing.T) {
1819	e, _, _ := remoteHomedEdge(t, []string{mirrorHost}, remoteCatchAll)
1820	e.issuer = func() (context.Context, bool) { return context.Background(), true }
1821	for _, host := range []string{remoteWildcardLabel, remoteCatchAll, "stranger.example"} {
1822	err := e.issuerHostPolicy()(context.Background(), host)
1823	if err == nil {
1824	t.Fatalf("%s: HostPolicy allowed an order", host)
1825	}
1826	if !strings.Contains(err.Error(), "not in registry") {
1827	t.Fatalf("%s: HostPolicy = %v, want the unchanged not-in-registry refusal", host, err)
1828	}
1829	}
1830	}
1831
1832	// Widening the NAMES a policy admits does not widen the BOXES that may order:
1833	// a non-issuer still refuses every name, remote-homed ones included.
1834	func TestHostPolicyStillGatesNonIssuer(t *testing.T) {
1835	e, _, _ := remoteHomedEdge(t, []string{mirrorHost}, remoteHomedHost)
1836	e.issuer = func() (context.Context, bool) { return nil, false }
1837	for _, host := range []string{remoteHomedHost, mirrorHost} {
1838	if err := e.issuerHostPolicy()(context.Background(), host); !errors.Is(err, errNotIssuer) {
1839	t.Fatalf("%s on a non-issuer = %v, want errNotIssuer", host, err)
1840	}
1841	}
1842	}
1843
1844	// Single-box deployments are byte-for-byte unchanged: no home map armed (or an
1845	// empty one), so deploymentHost reduces to Allowed.
1846	func TestDeploymentHostIsAllowedOnASingleBoxDeployment(t *testing.T) {
1847	e := testIssuerEdge(mirrorHost) // e.homes never stored: nil pointer
1848	if !e.deploymentHost(mirrorHost) || e.deploymentHost("stranger.example") {
1849	t.Fatal("with no home map at all, deploymentHost must be exactly Allowed")
1850	}
1851	e2, _, _ := remoteHomedEdge(t, []string{mirrorHost}) // map armed, folds to nothing
1852	if !e2.deploymentHost(mirrorHost) || e2.deploymentHost("stranger.example") {
1853	t.Fatal("with an empty home map, deploymentHost must be exactly Allowed")
1854	}
1855	}
1856
1857	// slowStore adds a fixed delay to every Get, standing in for a box that
1858	// reads the deployment bucket from the other side of the planet.
1859	type slowStore struct {
1860	release.Store
1861	delay time.Duration
1862	mu sync.Mutex
1863	peak int // greatest number of gets in flight at once
1864	cur int
1865	peakGoroutines int
1866	}
1867
1868	func (s *slowStore) Get(ctx context.Context, key string) ([]byte, string, error) {
1869	s.mu.Lock()
1870	s.cur++
1871	if s.cur > s.peak {
1872	s.peak = s.cur
1873	}
1874	// Sampled while the queue is deepest, which is what tells a fixed worker
1875	// pool from one goroutine per object: the first is flat in the number of
1876	// objects, the second is not.
1877	if g := runtime.NumGoroutine(); g > s.peakGoroutines {
1878	s.peakGoroutines = g
1879	}
1880	s.mu.Unlock()
1881	time.Sleep(s.delay)
1882	s.mu.Lock()
1883	s.cur--
1884	s.mu.Unlock()
1885	return s.Store.Get(ctx, key)
1886	}
1887
1888	// TestCertMirrorRefreshFetchesConcurrently is amadan #118's second half as a
1889	// test. The mirror's boot refresh runs BEFORE either listener binds, so its
1890	// cost is the edge's time-to-serve; the self-update gate allows 20s for the
1891	// edge to answer on loopback. Serially, the pass costs count x latency, and
1892	// the Sydney box (101 certs, ~250ms to eu-west-1) therefore took ~25s to
1893	// start serving and failed every gate from 2026-09-07 to 2026-09-19.
1894	//
1895	// The property is that the pass does NOT cost count x latency. Asserted as
1896	// wall-clock against a deliberately slow store, and by the peak number of
1897	// gets in flight, so a change that quietly re-serialises it is caught by
1898	// both.
1899	func TestCertMirrorRefreshFetchesConcurrently(t *testing.T) {
1900	const (
1901	certs = 40
1902	delay = 20 * time.Millisecond
1903	)
1904	inner := release.NewDirStore(t.TempDir())
1905	for i := range certs {
1906	putBucket(t, inner, fmt.Sprintf("host%02d.example.com", i), []byte("PEM"))
1907	}
1908	s := &slowStore{Store: inner, delay: delay}
1909	m := newCertMirror(s, nil)
1910
1911	base := runtime.NumGoroutine()
1912	start := time.Now()
1913	if err := m.Refresh(context.Background()); err != nil {
1914	t.Fatal(err)
1915	}
1916	took := time.Since(start)
1917
1918	// Serial would be at least certs*delay (800ms).
1919	if serial := certs * delay; took > serial/2 {
1920	t.Errorf("refresh took %s, serial would be ~%s — the gets are not concurrent", took, serial)
1921	}
1922	// The wall-clock bound above is weak on its own: it admits any
1923	// concurrency of about three or more, so a silent drop from ten to four
1924	// would pass it while putting a far-from-bucket box back over the update
1925	// gate.
1926	//
1927	// The floor is a LITERAL, deliberately. Deriving it from
1928	// certRefreshConcurrency makes the assertion move with the thing it is
1929	// meant to pin, and a change of the constant to 4 then passes — which is
1930	// exactly what happened to the first version of this line. Eight is the
1931	// arithmetic that matters: the update gate allows 20s, a far box is
1932	// ~250ms a round trip, so the pass must not exceed ~80 waves, and at a
1933	// few hundred published certs that needs most of ten in flight.
1934	const floor = 8
1935	if s.peak < floor {
1936	t.Errorf("peak gets in flight = %d, want at least %d — too little concurrency to fit the update gate on a far-from-bucket box", s.peak, floor)
1937	}
1938	if s.peak > certRefreshConcurrency {
1939	t.Errorf("peak gets in flight = %d, above the %d bound", s.peak, certRefreshConcurrency)
1940	}
1941	// And the pool is FIXED: goroutines must not scale with the number of
1942	// objects. A goroutine per object costs one stack per cert before a
1943	// single one is read, which is the shape that breaks at a million certs
1944	// rather than at a hundred.
1945	if s.peakGoroutines > base+certRefreshConcurrency+8 {
1946	t.Errorf("peak goroutines = %d with %d baseline and %d objects — the pool is not fixed", s.peakGoroutines, base, certs)
1947	}
1948	// And it still mirrors everything it was asked for.
1949	for i := range certs {
1950	if _, ok := m.Cert(fmt.Sprintf("host%02d.example.com", i)); !ok {
1951	t.Fatalf("host%02d.example.com missing from the mirror", i)
1952	}
1953	}
1954	}
1955
1956	// ---- #336: the change index gates the mirror's reads -------------------------
1957
1958	// changeIndexStore is a DirStore with the change index on that records every
1959	// read by method and key. Embedding the concrete store forwards
1960	// ConditionalGetter and ETagLister, so the mirror and the watch see the store
1961	// production hands them, and every write bumps its scope as in production.
1962	type changeIndexStore struct {
1963	*release.DirStore
1964	mu sync.Mutex
1965	calls []string
1966	failGet string // Get of exactly this key fails
1967	// block, when set, parks every GetIfChanged until it is closed: a hung
1968	// index GET holding the shared watch's poll lock. entered says one has.
1969	block chan struct{}
1970	entered chan struct{}
1971	}
1972
1973	// blockIndex makes the next index reads hang; closing the returned channel
1974	// (done at cleanup too) releases them.
1975	func (s *changeIndexStore) blockIndex(t *testing.T) (entered <-chan struct{}, release func()) {
1976	t.Helper()
1977	s.mu.Lock()
1978	s.block, s.entered = make(chan struct{}), make(chan struct{}, 16)
1979	block, ent := s.block, s.entered
1980	s.mu.Unlock()
1981	var once sync.Once
1982	release = func() {
1983	once.Do(func() {
1984	s.mu.Lock()
1985	s.block = nil
1986	s.mu.Unlock()
1987	close(block)
1988	})
1989	}
1990	t.Cleanup(release)
1991	return ent, release
1992	}
1993
1994	func (s *changeIndexStore) setFailGet(key string) { s.mu.Lock(); s.failGet = key; s.mu.Unlock() }
1995
1996	func newChangeIndexStore(t *testing.T) *changeIndexStore {
1997	t.Helper()
1998	d := release.NewDirStore(t.TempDir())
1999	d.EnableChangeIndex()
2000	return &changeIndexStore{DirStore: d}
2001	}
2002
2003	func (s *changeIndexStore) note(method, key string) {
2004	s.mu.Lock()
2005	s.calls = append(s.calls, method+" "+key)
2006	s.mu.Unlock()
2007	}
2008
2009	func (s *changeIndexStore) reset() { s.mu.Lock(); s.calls = nil; s.mu.Unlock() }
2010
2011	// count is how many recorded calls of method name a key under prefix.
2012	func (s *changeIndexStore) count(method, prefix string) int {
2013	s.mu.Lock()
2014	defer s.mu.Unlock()
2015	n := 0
2016	for _, c := range s.calls {
2017	if strings.HasPrefix(c, method+" "+prefix) {
2018	n++
2019	}
2020	}
2021	return n
2022	}
2023
2024	func (s *changeIndexStore) Get(ctx context.Context, key string) ([]byte, string, error) {
2025	s.note("Get", key)
2026	s.mu.Lock()
2027	fail := s.failGet != "" && s.failGet == key
2028	s.mu.Unlock()
2029	if fail {
2030	return nil, "", errors.New("get refused")
2031	}
2032	return s.DirStore.Get(ctx, key)
2033	}
2034
2035	func (s *changeIndexStore) GetIfChanged(ctx context.Context, key, etag string) ([]byte, string, bool, error) {
2036	s.note("GetIfChanged", key)
2037	s.mu.Lock()
2038	block, entered := s.block, s.entered
2039	s.mu.Unlock()
2040	if block != nil {
2041	entered <- struct{}{}
2042	<-block
2043	}
2044	return s.DirStore.GetIfChanged(ctx, key, etag)
2045	}
2046
2047	func (s *changeIndexStore) List(ctx context.Context, prefix string) ([]string, error) {
2048	s.note("List", prefix)
2049	return s.DirStore.List(ctx, prefix)
2050	}
2051
2052	func (s *changeIndexStore) ListETags(ctx context.Context, prefix string) ([]release.KeyETag, error) {
2053	s.note("ListETags", prefix)
2054	return s.DirStore.ListETags(ctx, prefix)
2055	}
2056
2057	// certReads is every bucket read the mirror made under control/certs/.
2058	func certReads(s *changeIndexStore) (list, listETags, get int) {
2059	return s.count("List", certPrefix), s.count("ListETags", certPrefix), s.count("Get", certPrefix)
2060	}
2061
2062	// armedMirror is a mirror over s with the change index armed: one watch over
2063	// the same store (the agent's, in production) and an injected clock.
2064	func armedMirror(t *testing.T, s *changeIndexStore) (*certMirror, *testClock) {
2065	t.Helper()
2066	w := release.NewChangeWatch(s)
2067	clk := &testClock{t: time.Date(2026, 10, 21, 12, 0, 0, 0, time.UTC)}
2068	m := newCertMirror(s, nil)
2069	m.watch = func() *release.ChangeWatch { return w }
2070	m.now = clk.now
2071	return m, clk
2072	}
2073
2074	func mirrorPoll(t *testing.T, m *certMirror) {
2075	t.Helper()
2076	if err := m.poll(context.Background()); err != nil {
2077	t.Fatalf("poll: %v", err)
2078	}
2079	}
2080
2081	// failingPutCache refuses every Put while fail is set, and counts the Puts
2082	// it lets through.
2083	type failingPutCache struct {
2084	autocert.Cache
2085	fail atomic.Bool
2086	puts atomic.Int32
2087	}
2088
2089	func (c *failingPutCache) Put(ctx context.Context, key string, data []byte) error {
2090	if c.fail.Load() {
2091	return errors.New("disk full")
2092	}
2093	c.puts.Add(1)
2094	return c.Cache.Put(ctx, key, data)
2095	}
2096
2097	// Final review R1 finding 7 (amadan #336): a copy into the local cert cache
2098	// that failed is retried on the next poll even when the index says nothing
2099	// moved, from the snapshot the mirror already holds, with no bucket read.
2100	// Once it lands, an unchanged poll writes nothing.
2101	func TestCertMirrorRetriesAFailedLocalCopyOnAnUnchangedPoll(t *testing.T) {
2102	t.Parallel()
2103	s := newChangeIndexStore(t)
2104	pem := hostCertPEM(t, "shop.example.com", time.Now().Add(80*24*time.Hour))
2105	putBucket(t, s, "shop.example.com", pem)
2106	m, clk := armedMirror(t, s)
2107	local := &failingPutCache{Cache: autocert.DirCache(t.TempDir())}
2108	m.local = local
2109	local.fail.Store(true)
2110	mirrorPoll(t, m)
2111	if _, err := local.Get(context.Background(), "shop.example.com"); err == nil {
2112	t.Fatal("fixture: the local copy landed while Puts fail")
2113	}
2114
2115	local.fail.Store(false)
2116	s.reset()
2117	clk.advance(time.Second)
2118	mirrorPoll(t, m)
2119	if body, err := local.Get(context.Background(), "shop.example.com"); err != nil || !bytes.Equal(body, pem) {
2120	t.Fatalf("an unchanged poll after a failed local copy left the local cache without it: %v", err)
2121	}
2122	if list, listETags, get := certReads(s); list+listETags+get != 0 {
2123	t.Fatalf("the retry read the bucket: list %d listETags %d get %d", list, listETags, get)
2124	}
2125	n := local.puts.Load()
2126	clk.advance(time.Second)
2127	mirrorPoll(t, m)
2128	if got := local.puts.Load(); got != n {
2129	t.Fatalf("a settled unchanged poll wrote the local cache again (%d puts, want %d)", got, n)
2130	}
2131	}
2132
2133	func TestCertMirrorSkipsWhenUnmoved(t *testing.T) {
2134	t.Parallel()
2135	s := newChangeIndexStore(t)
2136	putBucket(t, s, "example.com", []byte("PEM"))
2137	putBucket(t, s, "shop.example.com", []byte("PEM2"))
2138	m, clk := armedMirror(t, s)
2139	mirrorPoll(t, m)
2140	if b, ok := m.Cert("example.com"); !ok || string(b) != "PEM" {
2141	t.Fatalf("first poll: mirror cert = %q, %v", b, ok)
2142	}
2143	s.reset()
2144	clk.advance(certMirrorInterval)
2145	mirrorPoll(t, m)
2146	if l, le, g := certReads(s); l+le+g != 0 {
2147	t.Fatalf("unmoved poll read the bucket: List %d, ListETags %d, Get %d; want none", l, le, g)
2148	}
2149	if n := s.count("GetIfChanged", release.ChangeIndexKey); n != 1 {
2150	t.Fatalf("unmoved poll: %d index reads, want 1", n)
2151	}
2152	if b, ok := m.Cert("shop.example.com"); !ok || string(b) != "PEM2" {
2153	t.Fatalf("after a skipped poll the mirror = %q, %v — want the same copy", b, ok)
2154	}
2155	}
2156
2157	func TestCertMirrorFetchesOnlyChanged(t *testing.T) {
2158	t.Parallel()
2159	s := newChangeIndexStore(t)
2160	local := newCountingCache(t)
2161	pemA := hostCertPEM(t, "a.example.com", time.Now().Add(60*24*time.Hour))
2162	pemB := hostCertPEM(t, "b.example.com", time.Now().Add(60*24*time.Hour))
2163	putBucket(t, s, "a.example.com", pemA)
2164	putBucket(t, s, "b.example.com", pemB)
2165	putBucket(t, s, "c.example.com", []byte("PEM-C"))
2166	m, clk := armedMirror(t, s)
2167	m.local = local
2168	mirrorPoll(t, m)
2169
2170	renewed := hostCertPEM(t, "b.example.com", time.Now().Add(90*24*time.Hour))
2171	putBucket(t, s, "b.example.com", renewed)
2172	s.reset()
2173	clk.advance(certMirrorInterval)
2174	mirrorPoll(t, m)
2175	l, le, g := certReads(s)
2176	if l != 0 || le != 1 || g != 1 {
2177	t.Fatalf("one rewritten cert: List %d, ListETags %d, Get %d; want 0, 1, 1", l, le, g)
2178	}
2179	if n := s.count("Get", certKey("b.example.com")); n != 1 {
2180	t.Fatalf("the one Get must be the rewritten cert's, got %d of it", n)
2181	}
2182	if b, ok := m.Cert("b.example.com"); !ok || !bytes.Equal(b, renewed) {
2183	t.Fatal("the rewritten cert must be served from the new snapshot")
2184	}
2185	for _, name := range []string{"a.example.com", "c.example.com"} {
2186	if _, ok := m.Cert(name); !ok {
2187	t.Fatalf("%s did not move and must stay in the snapshot", name)
2188	}
2189	}
2190	if got, err := localCert(t, local, "b.example.com"); err != nil || !bytes.Equal(got, renewed) {
2191	t.Fatalf("a changed cert must still be copied into the local cache: %v", err)
2192	}
2193	if m.snapshot().certs["b.example.com"] == nil {
2194	t.Fatal("the changed cert must be parsed into the new snapshot")
2195	}
2196	}
2197
2198	func TestCertMirrorDropsRemovedKey(t *testing.T) {
2199	t.Parallel()
2200	const token = "0123456789abcdefghij0123456789abcdefghijkl"
2201	s := newChangeIndexStore(t)
2202	putBucket(t, s, "example.com", []byte("PEM"))
2203	putBucket(t, s, token+httpTokenSuffix, []byte("resp"))
2204	m, clk := armedMirror(t, s)
2205	mirrorPoll(t, m)
2206	if _, ok := m.Challenge(token); !ok {
2207	t.Fatal("precondition: the live token is mirrored")
2208	}
2209	if err := s.Delete(context.Background(), certKey(token+httpTokenSuffix)); err != nil {
2210	t.Fatal(err)
2211	}
2212	s.reset()
2213	clk.advance(certMirrorInterval)
2214	mirrorPoll(t, m)
2215	if _, ok := m.Challenge(token); ok {
2216	t.Fatal("a spent challenge token must leave the snapshot when its key leaves the listing")
2217	}
2218	if b, ok := m.Cert("example.com"); !ok || string(b) != "PEM" {
2219	t.Fatalf("the unmoved cert = %q, %v — want it kept", b, ok)
2220	}
2221	if l, le, g := certReads(s); l != 0 || le != 1 || g != 0 {
2222	t.Fatalf("a deletion: List %d, ListETags %d, Get %d; want 0, 1, 0", l, le, g)
2223	}
2224	}
2225
2226	func TestCertMirrorFullRebuildAtBackstop(t *testing.T) {
2227	t.Parallel()
2228	s := newChangeIndexStore(t)
2229	putBucket(t, s, "example.com", []byte("PEM"))
2230	putBucket(t, s, "shop.example.com", []byte("PEM2"))
2231	m, clk := armedMirror(t, s)
2232	mirrorPoll(t, m)
2233
2234	// Inside the jitter's lower bound (-10%): still trusted.
2235	s.reset()
2236	clk.advance(release.ChangeIndexBackstop*9/10 - time.Minute)
2237	mirrorPoll(t, m)
2238	if l, le, g := certReads(s); l+le+g != 0 {
2239	t.Fatalf("a young snapshot with the index unmoved: List %d, ListETags %d, Get %d; want none", l, le, g)
2240	}
2241
2242	// Past the jitter's upper bound (+10%): a full rebuild, index unmoved.
2243	s.reset()
2244	clk.advance(release.ChangeIndexBackstop*2/10 + 2*time.Minute)
2245	mirrorPoll(t, m)
2246	if l, le, g := certReads(s); l != 1 || le != 0 || g != 2 {
2247	t.Fatalf("at the backstop: List %d, ListETags %d, Get %d; want the full rebuild 1, 0, 2", l, le, g)
2248	}
2249
2250	// The rebuild resets the age: the next unmoved poll reads nothing.
2251	s.reset()
2252	clk.advance(certMirrorInterval)
2253	mirrorPoll(t, m)
2254	if l, le, g := certReads(s); l+le+g != 0 {
2255	t.Fatalf("after the backstop rebuild: List %d, ListETags %d, Get %d; want none", l, le, g)
2256	}
2257	}
2258
2259	func TestCertMirrorBootRefreshUnchanged(t *testing.T) {
2260	t.Parallel()
2261	s := newChangeIndexStore(t)
2262	putBucket(t, s, "example.com", []byte("PEM"))
2263	putBucket(t, s, "shop.example.com", []byte("PEM2"))
2264	m, clk := armedMirror(t, s)
2265	// The agent's tick has already polled the shared watch.
2266	if _, err := m.watch().Poll(context.Background()); err != nil {
2267	t.Fatal(err)
2268	}
2269	s.reset()
2270	// The boot path: a full read, whatever the index says, and twice over.
2271	for i := 0; i < 2; i++ {
2272	if err := m.Refresh(context.Background()); err != nil {
2273	t.Fatal(err)
2274	}
2275	}
2276	if l, le, g := certReads(s); l != 2 || le != 0 || g != 4 {
2277	t.Fatalf("two boot refreshes: List %d, ListETags %d, Get %d; want 2, 0, 4", l, le, g)
2278	}
2279	if n := s.count("GetIfChanged", ""); n != 0 {
2280	t.Fatalf("the boot refresh must not poll the index (%d reads): it runs before the listeners bind", n)
2281	}
2282	// The loop's first poll trusts the boot read.
2283	s.reset()
2284	clk.advance(certMirrorInterval)
2285	mirrorPoll(t, m)
2286	if l, le, g := certReads(s); l+le+g != 0 {
2287	t.Fatalf("first poll after boot, index unmoved: List %d, ListETags %d, Get %d; want none", l, le, g)
2288	}
2289	}
2290
2291	// Disarmed (no watch handed down, or the agent never armed one), every poll
2292	// is today's full Refresh.
2293	func TestCertMirrorDisarmedPollIsAFullRefresh(t *testing.T) {
2294	t.Parallel()
2295	s := newChangeIndexStore(t)
2296	putBucket(t, s, "example.com", []byte("PEM"))
2297	for _, watch := range []func() *release.ChangeWatch{nil, func() *release.ChangeWatch { return nil }} {
2298	m := newCertMirror(s, nil)
2299	m.watch = watch
2300	for i := 0; i < 2; i++ {
2301	s.reset()
2302	mirrorPoll(t, m)
2303	if l, le, g := certReads(s); l != 1 || le != 0 || g != 1 {
2304	t.Fatalf("disarmed poll %d: List %d, ListETags %d, Get %d; want 1, 0, 1", i, l, le, g)
2305	}
2306	if n := s.count("GetIfChanged", ""); n != 0 {
2307	t.Fatalf("disarmed poll read the index %d time(s)", n)
2308	}
2309	}
2310	}
2311	}
2312
2313	// A changed cert whose Get fails keeps its previous copy (degrade, don't
2314	// die), and the next poll reads it again with the index unmoved.
2315	func TestCertMirrorFailedGetOfAChangedCertKeepsThePreviousCopy(t *testing.T) {
2316	t.Parallel()
2317	s := newChangeIndexStore(t)
2318	putBucket(t, s, "example.com", []byte("PEM"))
2319	putBucket(t, s, "shop.example.com", []byte("PEM2"))
2320	m, clk := armedMirror(t, s)
2321	mirrorPoll(t, m)
2322
2323	putBucket(t, s, "example.com", []byte("PEM-RENEWED"))
2324	putBucket(t, s, "new.example.com", []byte("PEM3"))
2325	s.setFailGet(certKey("example.com"))
2326	clk.advance(certMirrorInterval)
2327	mirrorPoll(t, m)
2328	if b, ok := m.Cert("example.com"); !ok || string(b) != "PEM" {
2329	t.Fatalf("a failed Get of a changed cert: mirror = %q, %v — want the previous copy", b, ok)
2330	}
2331	if b, ok := m.Cert("new.example.com"); !ok || string(b) != "PEM3" {
2332	t.Fatalf("the other changed cert = %q, %v — want it landed", b, ok)
2333	}
2334
2335	s.setFailGet("")
2336	s.reset()
2337	clk.advance(certMirrorInterval)
2338	mirrorPoll(t, m)
2339	if b, ok := m.Cert("example.com"); !ok || string(b) != "PEM-RENEWED" {
2340	t.Fatalf("the retry: mirror = %q, %v — want the renewed copy", b, ok)
2341	}
2342	if l, le, g := certReads(s); l != 0 || le != 1 || g != 1 {
2343	t.Fatalf("the retry: List %d, ListETags %d, Get %d; want 0, 1, 1", l, le, g)
2344	}
2345	s.reset()
2346	clk.advance(certMirrorInterval)
2347	mirrorPoll(t, m)
2348	if l, le, g := certReads(s); l+le+g != 0 {
2349	t.Fatalf("settled: List %d, ListETags %d, Get %d; want none", l, le, g)
2350	}
2351	}
2352
2353	// The scopes the mirror and the home map gate on are the ones their prefixes'
2354	// writes bump.
2355	func TestCertMirrorAndHomeMapScopesMatchTheirPrefixes(t *testing.T) {
2356	t.Parallel()
2357	if got, ok := release.ChangeScope(certKey("example.com")); !ok || got != certScope {
2358	t.Fatalf("a cert write bumps %q (%v), the mirror gates on %q", got, ok, certScope)
2359	}
2360	if got, ok := release.ChangeScope(edgesPrefix + "box.json"); !ok || got != edgesScope {
2361	t.Fatalf("an edges-file write bumps %q (%v), the home map gates on %q", got, ok, edgesScope)
2362	}
2363	}
2364
2365	// WithChangeWatch reaches both loops, resolved per poll: a watch the agent
2366	// arms after the edge started is still seen.
2367	func TestCertMirrorWithChangeWatchReachesTheMirrorAndHomeMap(t *testing.T) {
2368	t.Parallel()
2369	s := newChangeIndexStore(t)
2370	var armed atomic.Pointer[release.ChangeWatch]
2371	e, err := New(openReg(t), nil)
2372	if err != nil {
2373	t.Fatal(err)
2374	}
2375	t.Cleanup(e.probers.Stop)
2376	e.staticStore = s
2377	WithChangeWatch(armed.Load)(e)
2378	m, hm := e.newCertMirror(nil), e.newHomeMap()
2379	if m.changeWatch() != nil || hm.changeWatch() != nil {
2380	t.Fatal("nothing armed yet: both loops must be disarmed")
2381	}
2382	w := release.NewChangeWatch(s)
2383	armed.Store(w)
2384	if m.changeWatch() != w || hm.changeWatch() != w {
2385	t.Fatal("a watch armed after the edge started must reach both loops")
2386	}
2387	}
2388
2389	// ONE bad object is not a bad poll, on the changed-only read too: a changed
2390	// cert that cannot be read keeps its previous copy, a token deleted in the
2391	// same window still leaves, and the poll succeeds (so the loop's backfill
2392	// runs), on every poll until the object reads again.
2393	func TestCertMirrorOneUnreadableChangedKeyIsNotABadPoll(t *testing.T) {
2394	t.Parallel()
2395	const token = "0123456789abcdefghij0123456789abcdefghijkl"
2396	s := newChangeIndexStore(t)
2397	putBucket(t, s, "example.com", []byte("PEM"))
2398	putBucket(t, s, token+httpTokenSuffix, []byte("resp"))
2399	m, clk := armedMirror(t, s)
2400	mirrorPoll(t, m)
2401
2402	putBucket(t, s, "example.com", []byte("PEM-RENEWED"))
2403	s.setFailGet(certKey("example.com"))
2404	if err := s.Delete(context.Background(), certKey(token+httpTokenSuffix)); err != nil {
2405	t.Fatal(err)
2406	}
2407	for i := 0; i < 2; i++ {
2408	s.reset()
2409	clk.advance(certMirrorInterval)
2410	if err := m.poll(context.Background()); err != nil {
2411	t.Fatalf("poll %d with one unreadable changed object: %v — want a successful partial poll", i, err)
2412	}
2413	if _, ok := m.Challenge(token); ok {
2414	t.Fatalf("poll %d: the deleted token must leave even while another object is unreadable", i)
2415	}
2416	if b, ok := m.Cert("example.com"); !ok || string(b) != "PEM" {
2417	t.Fatalf("poll %d: the unreadable cert = %q, %v — want the previous copy", i, b, ok)
2418	}
2419	if n := s.count("Get", certKey("example.com")); n != 1 {
2420	t.Fatalf("poll %d: %d Get(s) of the unreadable cert, want it retried once", i, n)
2421	}
2422	}
2423	}
2424
2425	// A hung index GET elsewhere in the process (the agent's tick holding the
2426	// shared watch's poll lock) costs the mirror its index read, not its poll:
2427	// it gives up on the watch and reads in full.
2428	func TestCertMirrorPollsPastAStuckWatch(t *testing.T) {
2429	t.Parallel()
2430	s := newChangeIndexStore(t)
2431	putBucket(t, s, "example.com", []byte("PEM"))
2432	m, clk := armedMirror(t, s)
2433	m.watchTimeout = 50 * time.Millisecond
2434	mirrorPoll(t, m)
2435
2436	entered, release := s.blockIndex(t)
2437	go func() { _, _ = m.watch().Poll(context.Background()) }()
2438	<-entered
2439	putBucket(t, s, "example.com", []byte("PEM-RENEWED"))
2440	clk.advance(certMirrorInterval)
2441	done := make(chan error, 1)
2442	go func() { done <- m.poll(context.Background()) }()
2443	select {
2444	case err := <-done:
2445	if err != nil {
2446	t.Fatalf("poll: %v", err)
2447	}
2448	case <-time.After(5 * time.Second):
2449	release()
2450	t.Fatal("the mirror's poll waited on a stuck watch")
2451	}
2452	if b, ok := m.Cert("example.com"); !ok || string(b) != "PEM-RENEWED" {
2453	t.Fatalf("a poll that could not read the index must read in full: mirror = %q, %v", b, ok)
2454	}
2455	}
2456