carlos / platform Public

Download this file

1	package edge
2
3	import (
4	"bufio"
5	"bytes"
6	"context"
7	"encoding/json"
8	"errors"
9	"fmt"
10	"io"
11	"net"
12	"net/http"
13	"net/http/httptest"
14	"os"
15	"path/filepath"
16	"slices"
17	"strings"
18	"sync/atomic"
19	"syscall"
20	"testing"
21	"time"
22
23	"github.com/carlosframework/platform/internal/provision"
24	"github.com/carlosframework/platform/internal/registry"
25	"github.com/carlosframework/platform/internal/release"
26	)
27
28	func openReg(t *testing.T) *registry.Registry {
29	t.Helper()
30	reg, err := registry.Open(filepath.Join(t.TempDir(), "registry.db"))
31	if err != nil {
32	t.Fatalf("open registry: %v", err)
33	}
34	return reg
35	}
36
37	// hostEchoBackend is an upstream that records the Host header it saw and
38	// answers a fixed body — enough to prove both routing and Host preservation.
39	func hostEchoBackend(t *testing.T, body string) (*httptest.Server, *string) {
40	t.Helper()
41	seenHost := new(string)
42	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
43	*seenHost = r.Host
44	io.WriteString(w, body)
45	}))
46	t.Cleanup(ts.Close)
47	return ts, seenHost
48	}
49
50	// getHost GETs the edge server with an explicit Host header, the way a real
51	// client reaches a routed hostname.
52	func getHost(t *testing.T, edgeURL, host string) (int, string) {
53	t.Helper()
54	req, err := http.NewRequest("GET", edgeURL+"/", nil)
55	if err != nil {
56	t.Fatalf("new request: %v", err)
57	}
58	req.Host = host
59	resp, err := http.DefaultClient.Do(req)
60	if err != nil {
61	t.Fatalf("GET %s (host %s): %v", edgeURL, host, err)
62	}
63	defer resp.Body.Close()
64	b, err := io.ReadAll(resp.Body)
65	if err != nil {
66	t.Fatalf("read body: %v", err)
67	}
68	return resp.StatusCode, string(b)
69	}
70
71	func TestRoutesHostToTCPUpstream(t *testing.T) {
72	backend, seenHost := hostEchoBackend(t, "hello from backend")
73	addr := backend.Listener.Addr().String()
74
75	reg := openReg(t)
76	if err := reg.Upsert(registry.Route{Host: "app.example.com", Kind: "service", Addr: addr, Status: "ready"}); err != nil {
77	t.Fatalf("upsert: %v", err)
78	}
79
80	e, err := New(reg, nil)
81	if err != nil {
82	t.Fatalf("New: %v", err)
83	}
84	srv := httptest.NewServer(e)
85	defer srv.Close()
86
87	code, body := getHost(t, srv.URL, "app.example.com")
88	if code != http.StatusOK {
89	t.Fatalf("status = %d, want 200", code)
90	}
91	if body != "hello from backend" {
92	t.Fatalf("body = %q, want backend body", body)
93	}
94	if *seenHost != "app.example.com" {
95	t.Fatalf("backend saw Host %q, want app.example.com (inbound Host must be preserved)", *seenHost)
96	}
97	if !e.Allowed("app.example.com") {
98	t.Fatalf("Allowed(app.example.com) = false, want true")
99	}
100	}
101
102	func TestSkipsNotReadyRoutes(t *testing.T) {
103	backend, _ := hostEchoBackend(t, "should never be reached")
104	addr := backend.Listener.Addr().String()
105
106	reg := openReg(t)
107	if err := reg.Upsert(registry.Route{Host: "pending.example.com", Kind: "service", Addr: addr, Status: "pending"}); err != nil {
108	t.Fatalf("upsert: %v", err)
109	}
110
111	e, err := New(reg, nil)
112	if err != nil {
113	t.Fatalf("New: %v", err)
114	}
115	srv := httptest.NewServer(e)
116	defer srv.Close()
117
118	code, _ := getHost(t, srv.URL, "pending.example.com")
119	if code != http.StatusNotFound {
120	t.Fatalf("status = %d, want 404 for a not-ready route", code)
121	}
122	// Allowed feeds the autocert HostPolicy: a not-ready route must not mint
123	// a certificate either.
124	if e.Allowed("pending.example.com") {
125	t.Fatalf("Allowed(pending.example.com) = true, want false")
126	}
127	}
128
129	func TestReloadPicksUpNewRoute(t *testing.T) {
130	backend, _ := hostEchoBackend(t, "late arrival")
131	addr := backend.Listener.Addr().String()
132
133	reg := openReg(t)
134	e, err := New(reg, nil)
135	if err != nil {
136	t.Fatalf("New: %v", err)
137	}
138	srv := httptest.NewServer(e)
139	defer srv.Close()
140
141	if code, _ := getHost(t, srv.URL, "new.example.com"); code != http.StatusNotFound {
142	t.Fatalf("pre-reload status = %d, want 404", code)
143	}
144
145	// Write via a SECOND handle on the same file — the cross-process shape
146	// (`carlos add` while the edge runs) that exposed the stale-WAL quirk.
147	reg2, err := registry.Open(reg.Path())
148	if err != nil {
149	t.Fatalf("open second handle: %v", err)
150	}
151	if err := reg2.Upsert(registry.Route{Host: "new.example.com", Kind: "service", Addr: addr, Status: "ready"}); err != nil {
152	t.Fatalf("upsert via second handle: %v", err)
153	}
154
155	if err := e.Reload(); err != nil {
156	t.Fatalf("Reload: %v", err)
157	}
158	code, body := getHost(t, srv.URL, "new.example.com")
159	if code != http.StatusOK || body != "late arrival" {
160	t.Fatalf("post-reload = %d %q, want 200 %q", code, body, "late arrival")
161	}
162	}
163
164	func TestUnixSocketUpstream(t *testing.T) {
165	sock := filepath.Join(t.TempDir(), "u.sock")
166	ln, err := net.Listen("unix", sock)
167	if err != nil {
168	// Sandboxed environments block unix-socket listeners (EPERM); that is
169	// an environment limit, not an edge bug.
170	if errors.Is(err, syscall.EPERM) || strings.Contains(err.Error(), "operation not permitted") {
171	t.Skipf("unix-socket listeners not permitted in this environment: %v", err)
172	}
173	t.Fatalf("listen unix: %v", err)
174	}
175	seenHost := new(string)
176	backend := &httptest.Server{
177	Listener: ln,
178	Config: &http.Server{Handler: http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
179	*seenHost = r.Host
180	io.WriteString(w, "over the socket")
181	})},
182	}
183	backend.Start()
184	defer backend.Close()
185
186	reg := openReg(t)
187	if err := reg.Upsert(registry.Route{Host: "sock.example.com", Kind: "instance", Socket: sock, Status: "ready"}); err != nil {
188	t.Fatalf("upsert: %v", err)
189	}
190
191	e, err := New(reg, nil)
192	if err != nil {
193	t.Fatalf("New: %v", err)
194	}
195	srv := httptest.NewServer(e)
196	defer srv.Close()
197
198	code, body := getHost(t, srv.URL, "sock.example.com")
199	if code != http.StatusOK {
200	t.Fatalf("status = %d, want 200", code)
201	}
202	if body != "over the socket" {
203	t.Fatalf("body = %q, want socket backend body", body)
204	}
205	if *seenHost != "sock.example.com" {
206	t.Fatalf("backend saw Host %q, want sock.example.com", *seenHost)
207	}
208	}
209
210	func TestFirstFingerprintObservationReloads(t *testing.T) {
211	backend, _ := hostEchoBackend(t, "seen on first poll")
212	addr := backend.Listener.Addr().String()
213
214	// Snapshot BEFORE the route exists: New's startup reload sees zero routes.
215	reg := openReg(t)
216	e, err := New(reg, nil)
217	if err != nil {
218	t.Fatalf("New: %v", err)
219	}
220	srv := httptest.NewServer(e)
221	defer srv.Close()
222
223	// The route lands after the startup reload but before the watcher starts —
224	// exactly the write a baseline-only first tick would silently absorb.
225	reg2, err := registry.Open(reg.Path())
226	if err != nil {
227	t.Fatalf("open second handle: %v", err)
228	}
229	if err := reg2.Upsert(registry.Route{Host: "first.example.com", Kind: "service", Addr: addr, Status: "ready"}); err != nil {
230	t.Fatalf("upsert: %v", err)
231	}
232
233	stop := make(chan struct{})
234	defer close(stop)
235	go e.Watch(stop)
236
237	// No second write: the FIRST observed fingerprint alone must reload.
238	deadline := time.Now().Add(6 * time.Second)
239	for time.Now().Before(deadline) {
240	if code, body := getHost(t, srv.URL, "first.example.com"); code == http.StatusOK {
241	if body != "seen on first poll" {
242	t.Fatalf("body = %q, want backend body", body)
243	}
244	return
245	}
246	time.Sleep(100 * time.Millisecond)
247	}
248	t.Fatalf("route never served: first fingerprint observation was baselined away instead of reloading")
249	}
250
251	// fakeActivator records the hibernation hook's calls.
252	type fakeActivator struct {
253	served int
254	retired []map[string]bool
255	}
256
257	func (f *fakeActivator) ServeHTTP(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool {
258	f.served++
259	proxy.ServeHTTP(w, r) // "wake" is instant; proxy straight through
260	return true
261	}
262
263	func (f *fakeActivator) RetireExcept(keep map[string]bool) {
264	f.retired = append(f.retired, keep)
265	}
266
267	// testEdgeWithStore builds an Edge with one route already loaded and static
268	// serving (the plumbing kind=status shares — see EnableStatic's doc comment)
269	// wired to a fresh, empty DirStore. It returns the store (so a test can Put
270	// bucket objects straight into it) and the Edge itself as an http.Handler:
271	// Edge.ServeHTTP dispatches on the request's Host header, so tests drive it
272	// with httptest.NewRecorder + httptest.NewRequest — no real listener, hence
273	// no unix-socket/EPERM sandbox concerns (contrast TestUnixSocketUpstream).
274	func testEdgeWithStore(t *testing.T, rt registry.Route) (release.Store, http.Handler) {
275	t.Helper()
276	reg := openReg(t)
277	if err := reg.Upsert(rt); err != nil {
278	t.Fatalf("upsert: %v", err)
279	}
280	e, err := New(reg, nil)
281	if err != nil {
282	t.Fatalf("New: %v", err)
283	}
284	store := release.NewDirStore(filepath.Join(t.TempDir(), "store"))
285	// The status class doesn't verify signatures (unlike static), but
286	// EnableStatic's signature still asks for a pubkey — any well-formed one
287	// works since it's never used on this path.
288	_, pubB64, err := release.GenerateKey()
289	if err != nil {
290	t.Fatalf("generate key: %v", err)
291	}
292	pub, err := release.ParsePublicKey(pubB64, "test")
293	if err != nil {
294	t.Fatalf("parse public key: %v", err)
295	}
296	e.EnableStatic(store, pub, filepath.Join(t.TempDir(), "cache"))
297	if err := e.Reload(); err != nil {
298	t.Fatalf("Reload: %v", err)
299	}
300	return store, e
301	}
302
303	func TestStatusRouteServesBucketBacked(t *testing.T) {
304	s, h := testEdgeWithStore(t, registry.Route{Host: "status.test", Kind: "status", Status: "ready"})
305	s.Put(context.Background(), "status/fleet.json", []byte(`{"generated_at":"x"}`), "")
306	s.Put(context.Background(), "status/index.html", []byte("<title>Fleet status</title>"), "")
307	for path, want := range map[string]string{"/": "Fleet status", "/fleet.json": "generated_at"} {
308	rec := httptest.NewRecorder()
309	h.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test"+path, nil))
310	if rec.Code != 200 || !strings.Contains(rec.Body.String(), want) {
311	t.Errorf("%s: %d %q", path, rec.Code, rec.Body.String())
312	}
313	}
314	rec := httptest.NewRecorder()
315	h.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test/boxes/anything", nil))
316	if rec.Code != 404 {
317	t.Errorf("non-allowlisted path must 404, got %d", rec.Code)
318	}
319	}
320
321	// TestStatusRouteContentTypes pins the three content types the brief names:
322	// a client (or the console iframe embedding the page) depends on these to
323	// render the HTML instead of downloading it, parse the JSON, and apply the
324	// stylesheet.
325	func TestStatusRouteContentTypes(t *testing.T) {
326	s, h := testEdgeWithStore(t, registry.Route{Host: "status.test", Kind: "status", Status: "ready"})
327	s.Put(context.Background(), "status/index.html", []byte("<html></html>"), "")
328	s.Put(context.Background(), "status/fleet.json", []byte(`{}`), "")
329	s.Put(context.Background(), "status/status.css", []byte("body{}"), "")
330
331	for path, wantPrefix := range map[string]string{
332	"/": "text/html",
333	"/fleet.json": "application/json",
334	"/status.css": "text/css",
335	} {
336	rec := httptest.NewRecorder()
337	h.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test"+path, nil))
338	if ct := rec.Header().Get("Content-Type"); !strings.HasPrefix(ct, wantPrefix) {
339	t.Errorf("%s: Content-Type = %q, want prefix %q", path, ct, wantPrefix)
340	}
341	}
342	}
343
344	// TestStatusRouteCacheServesStaleOnStoreError proves the two halves of the
345	// per-key cache's contract: inside the TTL, a mutated store object is NOT
346	// picked up (the cache, not the bucket, is the source of truth); after the
347	// TTL, a store error serves the last-fetched copy rather than failing the
348	// request — the status page outliving a bucket blip is the point (see the
349	// statusHandler doc comment in edge.go).
350	func TestStatusRouteCacheServesStaleOnStoreError(t *testing.T) {
351	rt := registry.Route{Host: "status.test", Kind: "status", Status: "ready"}
352	reg := openReg(t)
353	if err := reg.Upsert(rt); err != nil {
354	t.Fatalf("upsert: %v", err)
355	}
356	e, err := New(reg, nil)
357	if err != nil {
358	t.Fatalf("New: %v", err)
359	}
360	storeDir := filepath.Join(t.TempDir(), "store")
361	store := release.NewDirStore(storeDir)
362	_, pubB64, err := release.GenerateKey()
363	if err != nil {
364	t.Fatalf("generate key: %v", err)
365	}
366	pub, err := release.ParsePublicKey(pubB64, "test")
367	if err != nil {
368	t.Fatalf("parse public key: %v", err)
369	}
370	e.EnableStatic(store, pub, filepath.Join(t.TempDir(), "cache"))
371	if err := e.Reload(); err != nil {
372	t.Fatalf("Reload: %v", err)
373	}
374	if _, err := store.Put(context.Background(), "status/fleet.json", []byte("first"), ""); err != nil {
375	t.Fatalf("seed put: %v", err)
376	}
377
378	sh := e.statusHandlerFor(rt).(*statusHandler)
379	sh.ttl = 20 * time.Millisecond
380	fakeNow := time.Now()
381	sh.now = func() time.Time { return fakeNow }
382
383	rec := httptest.NewRecorder()
384	sh.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test/fleet.json", nil))
385	if rec.Code != 200 || rec.Body.String() != "first" {
386	t.Fatalf("first fetch = %d %q, want 200 %q", rec.Code, rec.Body.String(), "first")
387	}
388
389	// Mutate the store underneath the cache, but stay inside the TTL: the
390	// cached copy must still be served, proving the cache isn't a pass-through.
391	if _, err := store.Put(context.Background(), "status/fleet.json", []byte("second"), etagOfFile(t, storeDir, "status/fleet.json")); err != nil {
392	t.Fatalf("mutate put: %v", err)
393	}
394	rec = httptest.NewRecorder()
395	sh.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test/fleet.json", nil))
396	if rec.Code != 200 || rec.Body.String() != "first" {
397	t.Fatalf("inside-TTL fetch = %d %q, want cached %q", rec.Code, rec.Body.String(), "first")
398	}
399
400	// Advance the clock seam past the TTL: the handler must re-read the store
401	// and pick up the mutation.
402	fakeNow = fakeNow.Add(21 * time.Millisecond)
403	rec = httptest.NewRecorder()
404	sh.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test/fleet.json", nil))
405	if rec.Code != 200 || rec.Body.String() != "second" {
406	t.Fatalf("post-TTL fetch = %d %q, want refreshed %q", rec.Code, rec.Body.String(), "second")
407	}
408
409	// Now break the store (remove the underlying file, simulating a bucket
410	// error) and advance past the TTL again: with a stale copy in hand, the
411	// handler must keep serving it rather than 503ing.
412	fakeNow = fakeNow.Add(21 * time.Millisecond)
413	if err := os.Remove(filepath.Join(storeDir, "status", "fleet.json")); err != nil {
414	t.Fatalf("remove backing file: %v", err)
415	}
416	rec = httptest.NewRecorder()
417	sh.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test/fleet.json", nil))
418	if rec.Code != 200 || rec.Body.String() != "second" {
419	t.Fatalf("stale-on-error fetch = %d %q, want stale %q", rec.Code, rec.Body.String(), "second")
420	}
421	}
422
423	// TestStatusRouteEmptyStoreServes503 covers the honest-failure half: a key
424	// that has NEVER been successfully fetched, hit with a store error (here:
425	// the object was never published), must 503 rather than fabricate a body.
426	func TestStatusRouteEmptyStoreServes503(t *testing.T) {
427	_, h := testEdgeWithStore(t, registry.Route{Host: "status.test", Kind: "status", Status: "ready"})
428	rec := httptest.NewRecorder()
429	h.ServeHTTP(rec, httptest.NewRequest("GET", "http://status.test/fleet.json", nil))
430	if rec.Code != http.StatusServiceUnavailable {
431	t.Fatalf("empty-store fetch = %d, want 503", rec.Code)
432	}
433	}
434
435	// etagOfFile reads a DirStore-backed key straight off disk to recover its
436	// current etag for a replace-only Put — the same read-etag-then-Put dance
437	// internal/status/fleet.go's putOverwrite does against the real interface;
438	// done directly here since the test wants to mutate underneath an existing
439	// cache, not through it.
440	func etagOfFile(t *testing.T, root, key string) string {
441	t.Helper()
442	_, etag, err := release.NewDirStore(root).Get(context.Background(), key)
443	if err != nil {
444	t.Fatalf("read etag for %s: %v", key, err)
445	}
446	return etag
447	}
448
449	func TestHibernatingRouteGoesThroughActivator(t *testing.T) {
450	backend, _ := hostEchoBackend(t, "woken")
451	addr := backend.Listener.Addr().String()
452
453	reg := openReg(t)
454	if err := reg.Upsert(registry.Route{Host: "hib.example.com", Kind: "instance", Addr: addr, Status: "ready", Hibernate: true}); err != nil {
455	t.Fatalf("upsert: %v", err)
456	}
457
458	act := &fakeActivator{}
459	e, err := New(reg, act)
460	if err != nil {
461	t.Fatalf("New: %v", err)
462	}
463	srv := httptest.NewServer(e)
464	defer srv.Close()
465
466	code, body := getHost(t, srv.URL, "hib.example.com")
467	if code != http.StatusOK || body != "woken" {
468	t.Fatalf("got %d %q, want 200 woken", code, body)
469	}
470	if act.served != 1 {
471	t.Fatalf("activator ServeHTTP called %d times, want 1", act.served)
472	}
473	// Reload reconciles the activator with the new route set.
474	if len(act.retired) == 0 {
475	t.Fatalf("RetireExcept never called on reload")
476	}
477	last := act.retired[len(act.retired)-1]
478	if !last["hib.example.com"] {
479	t.Fatalf("RetireExcept keep-set %v missing hib.example.com", last)
480	}
481	}
482
483	// refusingActivator fails the test immediately if ever invoked — used to
484	// prove a cache hit never reaches the activator at all.
485	type refusingActivator struct{ t *testing.T }
486
487	func (r refusingActivator) ServeHTTP(w http.ResponseWriter, req *http.Request, rt registry.Route, proxy http.Handler) bool {
488	r.t.Fatal("activator invoked on what should have been a cache hit — this defeats the whole point of caching")
489	return true
490	}
491
492	func (r refusingActivator) RetireExcept(keep map[string]bool) {}
493
494	func TestServeHTTPCacheHitNeverInvokesActivator(t *testing.T) {
495	reg := openReg(t)
496	// Addr is deliberately a bogus, never-dialed address: if the cache
497	// check is ever bypassed, proxying to it fails loudly instead of
498	// silently succeeding, which would mask the bug.
499	if err := reg.Upsert(registry.Route{Host: "cached.example.com", Kind: "instance", Addr: "127.0.0.1:1", Status: "ready", Hibernate: true}); err != nil {
500	t.Fatalf("upsert: %v", err)
501	}
502	e, err := New(reg, refusingActivator{t: t})
503	if err != nil {
504	t.Fatalf("New: %v", err)
505	}
506	e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
507	key := CacheKey{Host: "cached.example.com", Method: "GET", Path: "/event"}
508	e.cache.Put(key, "", &CachedResponse{
509	StatusCode: 200,
510	Header: http.Header{"Content-Type": {"text/html"}},
511	Body: []byte("cached page"),
512	StoredAt: time.Now(),
513	TTL: time.Minute,
514	})
515
516	rec := httptest.NewRecorder()
517	e.ServeHTTP(rec, httptest.NewRequest("GET", "http://cached.example.com/event", nil))
518
519	if rec.Code != 200 {
520	t.Fatalf("status = %d, want 200", rec.Code)
521	}
522	if rec.Body.String() != "cached page" {
523	t.Fatalf("body = %q, want %q", rec.Body.String(), "cached page")
524	}
525	}
526
527	func TestServeHTTPCacheMissPopulatesCacheFromResponse(t *testing.T) {
528	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
529	w.Header().Set("Cache-Control", "max-age=60")
530	w.WriteHeader(200)
531	w.Write([]byte("fresh page"))
532	}))
533	t.Cleanup(backend.Close)
534
535	reg := openReg(t)
536	if err := reg.Upsert(registry.Route{Host: "miss.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil {
537	t.Fatalf("upsert: %v", err)
538	}
539	e, err := New(reg, nil)
540	if err != nil {
541	t.Fatalf("New: %v", err)
542	}
543	e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
544
545	rec1 := httptest.NewRecorder()
546	e.ServeHTTP(rec1, httptest.NewRequest("GET", "http://miss.example.com/event", nil))
547	if rec1.Body.String() != "fresh page" {
548	t.Fatalf("first response body = %q, want %q", rec1.Body.String(), "fresh page")
549	}
550
551	// Close the backend so a second real request would fail to dial —
552	// proves the second request is answered from cache, not the backend.
553	backend.Close()
554	rec2 := httptest.NewRecorder()
555	e.ServeHTTP(rec2, httptest.NewRequest("GET", "http://miss.example.com/event", nil))
556	if rec2.Body.String() != "fresh page" {
557	t.Fatalf("second response body = %q, want %q (should be served from cache)", rec2.Body.String(), "fresh page")
558	}
559	}
560
561	func TestServeHTTPNeverCachesNon2xx(t *testing.T) {
562	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
563	w.Header().Set("Cache-Control", "max-age=60")
564	w.WriteHeader(500)
565	w.Write([]byte("boom"))
566	}))
567	t.Cleanup(backend.Close)
568
569	reg := openReg(t)
570	if err := reg.Upsert(registry.Route{Host: "err.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil {
571	t.Fatalf("upsert: %v", err)
572	}
573	e, err := New(reg, nil)
574	if err != nil {
575	t.Fatalf("New: %v", err)
576	}
577	e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
578
579	e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://err.example.com/event", nil))
580
581	if _, ok := e.cache.get(CacheKey{Host: "err.example.com", Method: "GET", Path: "/event"}); ok {
582	t.Fatal("a 500 response must never be cached")
583	}
584	}
585
586	// varyBackendEdge wires an Edge in front of a backend that echoes the request's
587	// Accept-Language and counts its own invocations — the counter is what proves a
588	// second request was answered from cache rather than re-proxied, which asserting
589	// on response bodies alone cannot do (a miss echoes the same body a hit would).
590	func varyBackendEdge(t *testing.T, host string, hits *atomic.Int64) *Edge {
591	t.Helper()
592	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
593	hits.Add(1)
594	w.Header().Set("Cache-Control", "max-age=60")
595	w.Header().Set("Vary", "Accept-Language")
596	w.WriteHeader(200)
597	w.Write([]byte("lang=" + r.Header.Get("Accept-Language")))
598	}))
599	t.Cleanup(backend.Close)
600
601	reg := openReg(t)
602	if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil {
603	t.Fatalf("upsert: %v", err)
604	}
605	e, err := New(reg, nil)
606	if err != nil {
607	t.Fatalf("New: %v", err)
608	}
609	e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
610	return e
611	}
612
613	// getLang issues a GET carrying Accept-Language: lang (omitted entirely when
614	// lang is "") and returns the body.
615	func getLang(t *testing.T, e *Edge, url, lang string) string {
616	t.Helper()
617	req := httptest.NewRequest("GET", url, nil)
618	if lang != "" {
619	req.Header.Set("Accept-Language", lang)
620	}
621	rec := httptest.NewRecorder()
622	e.ServeHTTP(rec, req)
623	return rec.Body.String()
624	}
625
626	// A Vary-carrying response must be both separated per variant AND actually
627	// re-served from cache on a repeat request. The backend-invocation count is
628	// the load-bearing assertion: pre-fix, ServeHTTP looked entries up with an
629	// empty VaryValue while maybeStore stored them under varyValue(request, vary),
630	// so every request was a miss — 4 backend hits, not 2 — while the bodies still
631	// differed per language, which is why the old body-comparison assertion passed
632	// with Vary support entirely broken.
633	func TestServeHTTPVaryAcceptLanguageProducesSeparateEntries(t *testing.T) {
634	var hits atomic.Int64
635	e := varyBackendEdge(t, "i18n.example.com", &hits)
636	const url = "http://i18n.example.com/event"
637
638	for _, want := range []struct{ lang, body string }{
639	{"en", "lang=en"}, {"en", "lang=en"}, {"fr", "lang=fr"}, {"fr", "lang=fr"},
640	} {
641	if got := getLang(t, e, url, want.lang); got != want.body {
642	t.Fatalf("Accept-Language %q: body = %q, want %q", want.lang, got, want.body)
643	}
644	}
645
646	if n := hits.Load(); n != 2 {
647	t.Fatalf("backend invoked %d times across 4 requests (en, en, fr, fr), want 2 — the repeat of each language must be served from cache", n)
648	}
649	}
650
651	// The core regression: a repeat of an identical request against a
652	// Vary-carrying response must hit. Here no Accept-Language is sent at all, so
653	// both requests resolve to the same (empty-valued) variant.
654	func TestServeHTTPVaryRepeatIdenticalRequestHitsCache(t *testing.T) {
655	var hits atomic.Int64
656	e := varyBackendEdge(t, "repeat.example.com", &hits)
657	const url = "http://repeat.example.com/event"
658
659	if got := getLang(t, e, url, ""); got != "lang=" {
660	t.Fatalf("first body = %q, want %q", got, "lang=")
661	}
662	if got := getLang(t, e, url, ""); got != "lang=" {
663	t.Fatalf("second body = %q, want %q", got, "lang=")
664	}
665	if n := hits.Load(); n != 1 {
666	t.Fatalf("backend invoked %d times across 2 identical requests, want 1 — the second must be a cache hit", n)
667	}
668	}
669
670	// A crawler pumping distinct Vary values at one URL must not be able to fill
671	// the cache with unreachable variants: the per-URL cap bounds how many a
672	// single URL can occupy, evicting the oldest as new ones arrive.
673	func TestServeHTTPVaryVariantCapBoundsEntriesPerURL(t *testing.T) {
674	var hits atomic.Int64
675	e := varyBackendEdge(t, "flood.example.com", &hits)
676	const url = "http://flood.example.com/event"
677
678	langs := make([]string, 4*maxVaryVariantsPerURL)
679	for i := range langs {
680	langs[i] = fmt.Sprintf("xx-%d", i)
681	getLang(t, e, url, langs[i])
682	}
683
684	uk := CacheKey{Host: "flood.example.com", Method: "GET", Path: "/event"}.urlKey()
685	e.cache.mu.Lock()
686	n := len(e.cache.byURL[uk])
687	e.cache.mu.Unlock()
688	if n > maxVaryVariantsPerURL {
689	t.Fatalf("one URL holds %d cached variants, want at most %d", n, maxVaryVariantsPerURL)
690	}
691
692	// The newest variant survives; the oldest was evicted to make room.
693	before := hits.Load()
694	getLang(t, e, url, langs[len(langs)-1])
695	if hits.Load() != before {
696	t.Fatal("the most recent Vary variant should still be cached")
697	}
698	getLang(t, e, url, langs[0])
699	if hits.Load() != before+1 {
700	t.Fatal("the oldest Vary variant should have been evicted by the per-URL cap")
701	}
702	}
703
704	// cacheProbeEdge wires an Edge whose backend answers with headers chosen per
705	// path by hdr — the shape every "must not be cached" rule test needs.
706	func cacheProbeEdge(t *testing.T, host string, hdr func(path string, h http.Header)) *Edge {
707	t.Helper()
708	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
709	hdr(r.URL.Path, w.Header())
710	w.WriteHeader(200)
711	w.Write([]byte("body for " + r.URL.Path))
712	}))
713	t.Cleanup(backend.Close)
714
715	reg := openReg(t)
716	if err := reg.Upsert(registry.Route{Host: host, Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil {
717	t.Fatalf("upsert: %v", err)
718	}
719	e, err := New(reg, nil)
720	if err != nil {
721	t.Fatalf("New: %v", err)
722	}
723	e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
724	return e
725	}
726
727	func cachedAt(e *Edge, host, path string) bool {
728	_, ok := e.cache.get(CacheKey{Host: host, Method: "GET", Path: path})
729	return ok
730	}
731
732	// Cache-Control: private means "for one visitor only" — caching it at the
733	// shared edge and replaying it to everyone else is a cross-visitor leak.
734	func TestServeHTTPRefusesCacheControlPrivate(t *testing.T) {
735	e := cacheProbeEdge(t, "priv.example.com", func(path string, h http.Header) {
736	if path == "/private" {
737	h.Set("Cache-Control", "private, max-age=600")
738	return
739	}
740	h.Set("Cache-Control", "max-age=600")
741	})
742	e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://priv.example.com/private", nil))
743	e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://priv.example.com/shared", nil))
744
745	if cachedAt(e, "priv.example.com", "/private") {
746	t.Fatal("a Cache-Control: private response must never be cached at the shared edge")
747	}
748	if !cachedAt(e, "priv.example.com", "/shared") {
749	t.Fatal("an ordinary max-age response must still be cached")
750	}
751	}
752
753	// A Set-Cookie response carries per-visitor state (a session, a CSRF token);
754	// caching it would hand that cookie to every later visitor.
755	func TestServeHTTPRefusesResponseWithSetCookie(t *testing.T) {
756	e := cacheProbeEdge(t, "cookie2.example.com", func(path string, h http.Header) {
757	h.Set("Cache-Control", "max-age=600")
758	if path == "/session" {
759	h.Set("Set-Cookie", "sid=abc123; Path=/; HttpOnly")
760	}
761	})
762	e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://cookie2.example.com/session", nil))
763	e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://cookie2.example.com/plain", nil))
764
765	if cachedAt(e, "cookie2.example.com", "/session") {
766	t.Fatal("a response carrying Set-Cookie must never be cached — the cookie would be replayed to other visitors")
767	}
768	if !cachedAt(e, "cookie2.example.com", "/plain") {
769	t.Fatal("a response without Set-Cookie must still be cached")
770	}
771	}
772
773	// A response to an Authorization-bearing request is presumed private (RFC 9111
774	// §3.5) unless the app explicitly marks it public.
775	func TestServeHTTPRefusesAuthorizedRequestUnlessPublic(t *testing.T) {
776	e := cacheProbeEdge(t, "auth.example.com", func(path string, h http.Header) {
777	if path == "/public" {
778	h.Set("Cache-Control", "public, max-age=600")
779	return
780	}
781	h.Set("Cache-Control", "max-age=600")
782	})
783	authGet := func(path string) {
784	req := httptest.NewRequest("GET", "http://auth.example.com"+path, nil)
785	req.Header.Set("Authorization", "Bearer secret-token")
786	e.ServeHTTP(httptest.NewRecorder(), req)
787	}
788	authGet("/account")
789	authGet("/public")
790	e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://auth.example.com/anon", nil))
791
792	if cachedAt(e, "auth.example.com", "/account") {
793	t.Fatal("a response to an Authorization-bearing request must not be cached without an explicit Cache-Control: public")
794	}
795	if !cachedAt(e, "auth.example.com", "/public") {
796	t.Fatal("Cache-Control: public overrides the Authorization concern (RFC 9111 §3.5) — this response should be cached")
797	}
798	if !cachedAt(e, "auth.example.com", "/anon") {
799	t.Fatal("an unauthenticated request's ordinary max-age response must still be cached")
800	}
801	}
802
803	// An app asking for a year of freshness would make a single lost purge
804	// effectively permanent; the edge clamps every TTL to its own ceiling so a
805	// missed purge always self-heals within a bounded window (spec decision 4).
806	func TestServeHTTPClampsMaxAgeToCeiling(t *testing.T) {
807	e := cacheProbeEdge(t, "ttl.example.com", func(path string, h http.Header) {
808	h.Set("Cache-Control", "max-age=31536000")
809	})
810	e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://ttl.example.com/event", nil))
811
812	resp, ok := e.cache.get(CacheKey{Host: "ttl.example.com", Method: "GET", Path: "/event"})
813	if !ok {
814	t.Fatal("expected the response to be cached")
815	}
816	if resp.TTL != DefaultCacheMaxTTL {
817	t.Fatalf("cached TTL = %s, want it clamped to %s", resp.TTL, DefaultCacheMaxTTL)
818	}
819	}
820
821	// RFC 9111 §5.2.2.10: s-maxage overrides max-age for a SHARED cache, and a
822	// private cache ignores it. That pair is the only way an origin can say "hold
823	// this at the edge for two minutes, and have every browser revalidate" — which
824	// is what makes a purge effective end to end rather than leaving a client-side
825	// tail the purge channel cannot reach. Reading max-age alone turned exactly
826	// that header ("s-maxage=120, max-age=0") into "don't cache at all", so the
827	// origins trying hardest to stay purgeable were the ones that silently lost
828	// edge caching (issue #403; tito PRO-4758/PRO-4759).
829	func TestServeHTTPPrefersSMaxageOverMaxAge(t *testing.T) {
830	cases := []struct {
831	path string
832	cc []string // Cache-Control, one entry per header line
833	ttl time.Duration // 0 = must not be cached at all
834	}{
835	{"/pair", []string{"s-maxage=120, max-age=0"}, 120 * time.Second},
836	{"/pair-reversed", []string{"max-age=0, s-maxage=120"}, 120 * time.Second},
837	{"/pair-split", []string{"s-maxage=120", "max-age=0"}, 120 * time.Second},
838	{"/pair-split-reversed", []string{"max-age=0", "s-maxage=120"}, 120 * time.Second},
839	{"/shared-only", []string{"s-maxage=120"}, 120 * time.Second},
840	{"/mixed-case", []string{"S-MaxAge=120, Max-Age=0"}, 120 * time.Second},
841	{"/spaced", []string{"s-maxage = 120 , max-age=0"}, 120 * time.Second},
842	{"/shared-shorter", []string{"s-maxage=30, max-age=600"}, 30 * time.Second},
843	{"/max-age-only", []string{"max-age=60"}, 60 * time.Second},
844	{"/clamped", []string{"s-maxage=31536000, max-age=0"}, DefaultCacheMaxTTL},
845	{"/shared-zero", []string{"s-maxage=0, max-age=600"}, 0},
846	{"/private", []string{"private, s-maxage=120"}, 0},
847	{"/no-store", []string{"no-store, s-maxage=120"}, 0},
848	{"/no-cache", []string{"s-maxage=120, no-cache"}, 0},
849	{"/malformed-shared", []string{"s-maxage=soon, max-age=600"}, 0},
850	{"/malformed-max-age", []string{"max-age=soon, s-maxage=120"}, 0},
851	}
852	byPath := map[string][]string{}
853	for _, c := range cases {
854	byPath[c.path] = c.cc
855	}
856	e := cacheProbeEdge(t, "smaxage.example.com", func(path string, h http.Header) {
857	for _, line := range byPath[path] {
858	h.Add("Cache-Control", line)
859	}
860	})
861	for _, c := range cases {
862	t.Run(c.path, func(t *testing.T) {
863	e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://smaxage.example.com"+c.path, nil))
864	resp, ok := e.cache.get(CacheKey{Host: "smaxage.example.com", Method: "GET", Path: c.path})
865	if c.ttl == 0 {
866	if ok {
867	t.Fatalf("Cache-Control %q was cached; it must be refused", c.cc)
868	}
869	return
870	}
871	if !ok {
872	t.Fatalf("Cache-Control %q was not cached; the edge should hold it for %s", c.cc, c.ttl)
873	}
874	if resp.TTL != c.ttl {
875	t.Fatalf("Cache-Control %q cached with TTL %s, want %s", c.cc, resp.TTL, c.ttl)
876	}
877	})
878	}
879	}
880
881	// The stored response is replayed to later visitors verbatim, so the browser
882	// half of the pair has to survive the edge: an origin that said max-age=0
883	// still gets a revalidating browser on a cache HIT, not the edge's own idea of
884	// the client lifetime. Nothing rewrites Cache-Control on the way downstream —
885	// which is the point of reading s-maxage rather than inventing a header.
886	func TestServeHTTPCacheHitReplaysOriginCacheControl(t *testing.T) {
887	e := cacheProbeEdge(t, "replay.example.com", func(path string, h http.Header) {
888	h.Set("Cache-Control", "s-maxage=120, max-age=0")
889	})
890	get := func(label string) {
891	t.Helper()
892	rec := httptest.NewRecorder()
893	e.ServeHTTP(rec, httptest.NewRequest("GET", "http://replay.example.com/page", nil))
894	if got := rec.Header().Get("Cache-Control"); got != "s-maxage=120, max-age=0" {
895	t.Fatalf("%s: Cache-Control = %q, want the origin's own header", label, got)
896	}
897	}
898	get("miss")
899	if !cachedAt(e, "replay.example.com", "/page") {
900	t.Fatal("s-maxage=120 should have been stored at the edge")
901	}
902	get("hit")
903	}
904
905	func TestServeHTTPRefusesVaryCookie(t *testing.T) {
906	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
907	w.Header().Set("Cache-Control", "max-age=60")
908	w.Header().Set("Vary", "Cookie")
909	w.WriteHeader(200)
910	w.Write([]byte("should not be cached"))
911	}))
912	t.Cleanup(backend.Close)
913
914	reg := openReg(t)
915	if err := reg.Upsert(registry.Route{Host: "cookie.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil {
916	t.Fatalf("upsert: %v", err)
917	}
918	e, err := New(reg, nil)
919	if err != nil {
920	t.Fatalf("New: %v", err)
921	}
922	e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
923
924	e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://cookie.example.com/event", nil))
925
926	if _, ok := e.cache.get(CacheKey{Host: "cookie.example.com", Method: "GET", Path: "/event"}); ok {
927	t.Fatal("a response Vary-ing on Cookie must never be cached — it would defeat per-visitor correctness")
928	}
929	}
930
931	// cachedVariants counts every cached variant of one URL, whatever VaryValue
932	// each was stored under. A "must not be cached" assertion has to ask this
933	// question rather than Get with an empty VaryValue: a response that varies is
934	// stored under a computed variant key, so the empty-key Get misses and the
935	// assertion passes whether or not the refusal actually fired.
936	func cachedVariants(e *Edge, host, path string) int {
937	uk := CacheKey{Host: host, Method: "GET", Path: path}.urlKey()
938	e.cache.mu.Lock()
939	defer e.cache.mu.Unlock()
940	return len(e.cache.byURL[uk])
941	}
942
943	// getWith issues a GET carrying one extra header and returns the recorder.
944	func getWith(t *testing.T, e *Edge, url, name, value string) *httptest.ResponseRecorder {
945	t.Helper()
946	req := httptest.NewRequest("GET", url, nil)
947	if name != "" {
948	req.Header.Set(name, value)
949	}
950	rec := httptest.NewRecorder()
951	e.ServeHTTP(rec, req)
952	return rec
953	}
954
955	// RFC 9110 §5.3: one header field may arrive as several lines, and that is
956	// semantically identical to a single comma-joined line — which is exactly what
957	// http.Header.Add produces and what layered middleware emits in practice
958	// (compression adds Vary: Accept-Encoding, the session layer separately adds
959	// Vary: Cookie). http.Header.Get returns only the FIRST line, so reading Vary
960	// that way made everything after it invisible: the Vary: Cookie refusal never
961	// fired and the entry was keyed on Accept-Encoding alone, so one visitor's
962	// cookie-scoped response was replayed to the next visitor.
963	func TestServeHTTPRefusesVaryCookieSplitAcrossHeaderLines(t *testing.T) {
964	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
965	w.Header().Set("Cache-Control", "max-age=60")
966	// Two separate Add calls, the standard shape for a list-valued header.
967	w.Header().Add("Vary", "Accept-Encoding")
968	w.Header().Add("Vary", "Cookie")
969	w.WriteHeader(200)
970	w.Write([]byte("dashboard for " + r.Header.Get("Cookie")))
971	}))
972	t.Cleanup(backend.Close)
973
974	reg := openReg(t)
975	if err := reg.Upsert(registry.Route{Host: "split.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil {
976	t.Fatalf("upsert: %v", err)
977	}
978	e, err := New(reg, nil)
979	if err != nil {
980	t.Fatalf("New: %v", err)
981	}
982	e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
983	const url = "http://split.example.com/dashboard"
984
985	alice := getWith(t, e, url, "Cookie", "sid=alice").Body.String()
986	if alice != "dashboard for sid=alice" {
987	t.Fatalf("alice body = %q, want %q", alice, "dashboard for sid=alice")
988	}
989	if n := cachedVariants(e, "split.example.com", "/dashboard"); n != 0 {
990	// Errorf, not Fatalf: the replay assertion below is the live
991	// consequence and is worth reporting in the same run.
992	t.Errorf("%d variants cached for a response that varies on Cookie, want 0 — a multi-line Vary is still a Vary", n)
993	}
994
995	bob := getWith(t, e, url, "Cookie", "sid=bob").Body.String()
996	if bob != "dashboard for sid=bob" {
997	t.Fatalf("bob was served %q — one visitor's cookie-scoped response was replayed to another", bob)
998	}
999	}
1000
1001	// The Vary refusals read the raw field value and compare case-insensitively,
1002	// so they fire on every spelling a response can use — including the ones the
1003	// canonicalization downstream would have tidied up anyway. Pinned separately
1004	// because canonicalization is the kind of change that invites moving the
1005	// refusals behind it, where a name the canonicalizer does not recognise (or
1006	// declines to rewrite) would slip past.
1007	func TestServeHTTPRefusesVaryCookieAndStarInAnySpelling(t *testing.T) {
1008	e := cacheProbeEdge(t, "spell.example.com", func(path string, h http.Header) {
1009	h.Set("Cache-Control", "max-age=60")
1010	switch path {
1011	case "/lower":
1012	h.Set("Vary", "accept-encoding, cookie")
1013	case "/upper":
1014	h.Add("Vary", "Accept-Language")
1015	h.Add("Vary", "COOKIE")
1016	case "/star":
1017	h.Set("Vary", "Accept-Encoding , * ")
1018	case "/ok":
1019	h.Set("Vary", "accept-encoding")
1020	}
1021	})
1022	for _, path := range []string{"/lower", "/upper", "/star", "/ok"} {
1023	e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://spell.example.com"+path, nil))
1024	}
1025	for _, path := range []string{"/lower", "/upper", "/star"} {
1026	if n := cachedVariants(e, "spell.example.com", path); n != 0 {
1027	t.Errorf("%s: %d variants cached, want 0 — Vary: Cookie and Vary: * must be refused whatever their spelling or line", path, n)
1028	}
1029	}
1030	if n := cachedVariants(e, "spell.example.com", "/ok"); n != 1 {
1031	t.Errorf("/ok: %d variants cached, want 1 — an ordinary Vary in lower case must still be cached", n)
1032	}
1033	}
1034
1035	// The same root cause on Cache-Control: with max-age=600 on one line and
1036	// private on the next, Header.Get saw only max-age and the private response
1037	// was cached at the shared edge — the cross-visitor leak the private rule
1038	// exists to prevent.
1039	func TestServeHTTPRefusesCacheControlPrivateSplitAcrossHeaderLines(t *testing.T) {
1040	e := cacheProbeEdge(t, "splitcc.example.com", func(path string, h http.Header) {
1041	if path == "/private" {
1042	h.Add("Cache-Control", "max-age=600")
1043	h.Add("Cache-Control", "private")
1044	return
1045	}
1046	h.Set("Cache-Control", "max-age=600")
1047	})
1048	e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://splitcc.example.com/private", nil))
1049	e.ServeHTTP(httptest.NewRecorder(), httptest.NewRequest("GET", "http://splitcc.example.com/shared", nil))
1050
1051	if n := cachedVariants(e, "splitcc.example.com", "/private"); n != 0 {
1052	t.Fatalf("%d entries cached for a response whose second Cache-Control line says private, want 0", n)
1053	}
1054	if !cachedAt(e, "splitcc.example.com", "/shared") {
1055	t.Fatal("an ordinary max-age response must still be cached")
1056	}
1057	}
1058
1059	// The same root cause once more, on the Authorization rule: a response to an
1060	// Authorization-bearing request is private per RFC 9111 §3.5 unless
1061	// Cache-Control says public — and "public" on a second header line counts.
1062	func TestServeHTTPAuthorizedPublicSplitAcrossHeaderLines(t *testing.T) {
1063	e := cacheProbeEdge(t, "splitauth.example.com", func(path string, h http.Header) {
1064	h.Add("Cache-Control", "max-age=600")
1065	h.Add("Cache-Control", "public")
1066	})
1067	req := httptest.NewRequest("GET", "http://splitauth.example.com/public", nil)
1068	req.Header.Set("Authorization", "Bearer secret-token")
1069	e.ServeHTTP(httptest.NewRecorder(), req)
1070
1071	if !cachedAt(e, "splitauth.example.com", "/public") {
1072	t.Fatal("Cache-Control: public on a second header line still overrides the Authorization concern")
1073	}
1074	}
1075
1076	// An app that emits Vary only for some requests — CORS middleware adding
1077	// Vary: Origin only when the request carries Origin, content negotiation
1078	// adding Vary: Accept only when it must — used to poison its own URL
1079	// permanently: the first Vary-carrying response recorded the Vary names for
1080	// that URL and nothing ever cleared them, so later non-Vary responses were
1081	// stored under the empty variant while every lookup kept resolving against the
1082	// remembered names. Store and lookup drifted apart, every ordinary request
1083	// missed, and on a hibernating route every miss is a wake — a remote,
1084	// unauthenticated wake-storm from one request.
1085	func TestServeHTTPConditionalVaryDoesNotPoisonURL(t *testing.T) {
1086	var hits atomic.Int64
1087	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1088	hits.Add(1)
1089	w.Header().Set("Cache-Control", "max-age=60")
1090	if origin := r.Header.Get("Origin"); origin != "" {
1091	w.Header().Set("Vary", "Origin")
1092	w.Header().Set("Access-Control-Allow-Origin", origin)
1093	}
1094	w.WriteHeader(200)
1095	w.Write([]byte("page"))
1096	}))
1097	t.Cleanup(backend.Close)
1098
1099	reg := openReg(t)
1100	if err := reg.Upsert(registry.Route{Host: "cors.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil {
1101	t.Fatalf("upsert: %v", err)
1102	}
1103	e, err := New(reg, nil)
1104	if err != nil {
1105	t.Fatalf("New: %v", err)
1106	}
1107	e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
1108	const url = "http://cors.example.com/page"
1109
1110	// One unauthenticated request on a cold URL, eliciting the Vary variant.
1111	getWith(t, e, url, "Origin", "https://crawler.example")
1112
1113	// Then ordinary traffic: the first is a legitimate miss, every one after
1114	// it must be served from cache.
1115	var last *httptest.ResponseRecorder
1116	for i := 0; i < 10; i++ {
1117	last = getWith(t, e, url, "", "")
1118	if got := last.Body.String(); got != "page" {
1119	t.Fatalf("ordinary request %d body = %q, want %q", i, got, "page")
1120	}
1121	}
1122	if n := hits.Load(); n != 2 {
1123	t.Fatalf("backend invoked %d times for 1 Origin request + 10 ordinary ones, want 2 — after the first ordinary miss every ordinary request must hit the cache", n)
1124	}
1125	if acao := last.Header().Get("Access-Control-Allow-Origin"); acao != "" {
1126	t.Fatalf("ordinary request served the Origin variant (Access-Control-Allow-Origin: %s)", acao)
1127	}
1128	}
1129
1130	// The same root cause on the Authorization rule, but in the read of the
1131	// REQUEST'S Authorization header rather than the response's Cache-Control:
1132	// http.Header.Get returns only the first line, so a request that splits
1133	// Authorization across two lines (an empty first line, the real credential on
1134	// the second — Go's HTTP server accepts exactly this from a client) made Get
1135	// see "" and the request looked unauthenticated. The RFC 9111 §3.5 refusal
1136	// never fired and a credentialed response got cached at the shared edge for
1137	// everyone.
1138	func TestServeHTTPRefusesAuthorizationSplitAcrossHeaderLines(t *testing.T) {
1139	e := cacheProbeEdge(t, "splitauthreq.example.com", func(path string, h http.Header) {
1140	if path == "/public" {
1141	h.Set("Cache-Control", "max-age=60, public")
1142	return
1143	}
1144	h.Set("Cache-Control", "max-age=60")
1145	})
1146
1147	req := httptest.NewRequest("GET", "http://splitauthreq.example.com/private", nil)
1148	req.Header.Add("Authorization", "")
1149	req.Header.Add("Authorization", "Bearer secret-token")
1150	e.ServeHTTP(httptest.NewRecorder(), req)
1151
1152	if n := cachedVariants(e, "splitauthreq.example.com", "/private"); n != 0 {
1153	t.Fatalf("%d variants cached for a request whose Authorization arrived on a second header line, want 0 — a multi-line Authorization is still an Authorization", n)
1154	}
1155
1156	// Positive control: the same split-Authorization request, but the
1157	// response says public — proving the refusal isn't just breaking caching
1158	// wholesale.
1159	pubReq := httptest.NewRequest("GET", "http://splitauthreq.example.com/public", nil)
1160	pubReq.Header.Add("Authorization", "")
1161	pubReq.Header.Add("Authorization", "Bearer secret-token")
1162	e.ServeHTTP(httptest.NewRecorder(), pubReq)
1163
1164	if !cachedAt(e, "splitauthreq.example.com", "/public") {
1165	t.Fatal("Cache-Control: public must still be cached even with a split-line Authorization present")
1166	}
1167	}
1168
1169	// RFC 9111 §4.1 / RFC 9110 §12.5.5: Vary: * means the response is NEVER
1170	// reusable from a shared cache — it signals that the response depends on
1171	// something not captured by request headers at all. The token-scanning loop
1172	// treated "*" as if it were an ordinary header name: varyValue computed the
1173	// constant string "*=;" for every request regardless of who they were, so one
1174	// variant was stored and replayed to everyone.
1175	func TestServeHTTPRefusesVaryStar(t *testing.T) {
1176	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1177	w.Header().Set("Cache-Control", "max-age=60")
1178	w.Header().Set("Vary", "*")
1179	w.WriteHeader(200)
1180	w.Write([]byte("dashboard for " + r.Header.Get("Cookie")))
1181	}))
1182	t.Cleanup(backend.Close)
1183
1184	reg := openReg(t)
1185	if err := reg.Upsert(registry.Route{Host: "varystar.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil {
1186	t.Fatalf("upsert: %v", err)
1187	}
1188	e, err := New(reg, nil)
1189	if err != nil {
1190	t.Fatalf("New: %v", err)
1191	}
1192	e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
1193	const url = "http://varystar.example.com/dashboard"
1194
1195	alice := getWith(t, e, url, "Cookie", "sid=alice").Body.String()
1196	if alice != "dashboard for sid=alice" {
1197	t.Fatalf("alice body = %q, want %q", alice, "dashboard for sid=alice")
1198	}
1199	if n := cachedVariants(e, "varystar.example.com", "/dashboard"); n != 0 {
1200	// Errorf, not Fatalf: the replay assertion below is the live
1201	// consequence and is worth reporting in the same run.
1202	t.Errorf("%d variants cached for a response carrying Vary: *, want 0 — Vary: * means never reusable from a shared cache", n)
1203	}
1204
1205	bob := getWith(t, e, url, "Cookie", "sid=bob").Body.String()
1206	if bob != "dashboard for sid=bob" {
1207	t.Fatalf("bob was served %q — one visitor's response was replayed to another despite Vary: *", bob)
1208	}
1209	}
1210
1211	// An app that emits Vary conditionally — Vary: Accept-Language when the
1212	// request negotiates a language, no Vary at all when it doesn't — used to
1213	// thrash its own cache entries. One Vary scheme was recorded per URL, so every
1214	// flip between the two response shapes rewrote that record and swept every
1215	// variant stored under the previous scheme. Worse than a miss: with the record
1216	// cleared, a language-negotiating request resolved to the EMPTY variant and was
1217	// served the no-Vary body, so the language the app negotiated silently
1218	// disappeared for everyone.
1219	//
1220	// With the Vary names stored per entry, the two schemes coexist: each request
1221	// shape matches the entry its own shape produced, both stay live, and only the
1222	// first of each is a miss.
1223	func TestServeHTTPConditionalVaryKeepsBothVariantsLive(t *testing.T) {
1224	var hits atomic.Int64
1225	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1226	hits.Add(1)
1227	w.Header().Set("Cache-Control", "max-age=60")
1228	lang := r.Header.Get("Accept-Language")
1229	if lang != "" {
1230	w.Header().Set("Vary", "Accept-Language")
1231	}
1232	w.WriteHeader(200)
1233	w.Write([]byte("lang=" + lang))
1234	}))
1235	t.Cleanup(backend.Close)
1236
1237	reg := openReg(t)
1238	if err := reg.Upsert(registry.Route{Host: "mixedvary.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready"}); err != nil {
1239	t.Fatalf("upsert: %v", err)
1240	}
1241	e, err := New(reg, nil)
1242	if err != nil {
1243	t.Fatalf("New: %v", err)
1244	}
1245	e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
1246	const url = "http://mixedvary.example.com/page"
1247
1248	// Alternate the two shapes. The first of each is a legitimate miss; every
1249	// repeat after that must be served from cache, with its OWN body.
1250	for i := 0; i < 6; i++ {
1251	if got := getLang(t, e, url, "en"); got != "lang=en" {
1252	t.Fatalf("round %d: language-negotiating request served %q, want %q", i, got, "lang=en")
1253	}
1254	if got := getLang(t, e, url, ""); got != "lang=" {
1255	t.Fatalf("round %d: plain request served %q, want %q", i, got, "lang=")
1256	}
1257	}
1258
1259	if n := hits.Load(); n != 2 {
1260	t.Fatalf("backend invoked %d times across 12 requests alternating two Vary schemes, want 2 — after one miss each, both variants must stay live and hit", n)
1261	}
1262	if n := cachedVariants(e, "mixedvary.example.com", "/page"); n != 2 {
1263	t.Fatalf("URL holds %d cached variants, want 2 — the varying and the non-varying response must coexist, not evict each other", n)
1264	}
1265	}
1266
1267	// hijackableRecorder embeds httptest.ResponseRecorder (which deliberately
1268	// does NOT implement http.Hijacker) and adds a no-op Hijack, standing in for
1269	// a real connection's ResponseWriter — the minimum shape needed to prove
1270	// ServeHTTP passes an Upgrade request's ResponseWriter through untouched
1271	// instead of wrapping it in recordingResponseWriter (which pre-fix had no
1272	// Hijack, breaking the stdlib reverse proxy's WebSocket-upgrade path).
1273	type hijackableRecorder struct {
1274	*httptest.ResponseRecorder
1275	}
1276
1277	func (h *hijackableRecorder) Hijack() (net.Conn, *bufio.ReadWriter, error) {
1278	return nil, nil, nil
1279	}
1280
1281	// hijackCheckingActivator fails the test if the ResponseWriter it's handed
1282	// doesn't satisfy http.Hijacker — proving ServeHTTP special-cased the
1283	// Upgrade request instead of wrapping it in recordingResponseWriter.
1284	type hijackCheckingActivator struct{ t *testing.T }
1285
1286	func (h hijackCheckingActivator) ServeHTTP(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool {
1287	if _, ok := w.(http.Hijacker); !ok {
1288	h.t.Fatal("ResponseWriter passed to activator for an Upgrade request does not implement http.Hijacker — recordingResponseWriter must not wrap upgrade requests")
1289	}
1290	return true
1291	}
1292
1293	func (h hijackCheckingActivator) RetireExcept(keep map[string]bool) {}
1294
1295	// TestServeHTTPUpgradeRequestBypassesRecording proves a Connection: Upgrade
1296	// request (a WebSocket handshake) to a hibernating route reaches the
1297	// activator with a still-Hijacker-capable ResponseWriter — i.e. ServeHTTP
1298	// never wraps it in recordingResponseWriter, which would break the stdlib
1299	// reverse proxy's hijack-based upgrade path and, with it, the open WebSocket
1300	// the activator relies on to keep the instance's idle clock from firing.
1301	func TestServeHTTPUpgradeRequestBypassesRecording(t *testing.T) {
1302	reg := openReg(t)
1303	// Addr is a bogus, never-dialed address — this test only needs to reach
1304	// the activator, never the real backend.
1305	if err := reg.Upsert(registry.Route{Host: "ws.example.com", Kind: "instance", Addr: "127.0.0.1:1", Status: "ready", Hibernate: true}); err != nil {
1306	t.Fatalf("upsert: %v", err)
1307	}
1308	e, err := New(reg, hijackCheckingActivator{t: t})
1309	if err != nil {
1310	t.Fatalf("New: %v", err)
1311	}
1312	e.EnableCache(NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes))
1313
1314	req := httptest.NewRequest("GET", "http://ws.example.com/socket", nil)
1315	req.Header.Set("Connection", "Upgrade")
1316	req.Header.Set("Upgrade", "websocket")
1317	rec := &hijackableRecorder{ResponseRecorder: httptest.NewRecorder()}
1318
1319	e.ServeHTTP(rec, req)
1320	}
1321
1322	// TestIsUpgradeRequestDetectsConnectionSplitAcrossHeaderLines is the last of
1323	// the four Header.Get-on-a-list-valued-header bugs found on this branch
1324	// (see joinHeader's doc comment): Connection is legitimately list-valued
1325	// (RFC 9110 §5.3), and a client that sends "Connection: keep-alive" and
1326	// "Connection: Upgrade" as two separate header lines is just as much an
1327	// upgrade request as one that sends them comma-joined on one line — but
1328	// http.Header.Get only ever returns the first line, so isUpgradeRequest
1329	// would silently treat the split form as an ordinary request and let it
1330	// fall through to the cache/recordingResponseWriter path instead of the
1331	// untouched-passthrough branch.
1332	func TestIsUpgradeRequestDetectsConnectionSplitAcrossHeaderLines(t *testing.T) {
1333	req := httptest.NewRequest("GET", "http://ws.example.com/socket", nil)
1334	req.Header.Add("Connection", "keep-alive")
1335	req.Header.Add("Connection", "Upgrade")
1336
1337	if !isUpgradeRequest(req) {
1338	t.Fatal("Connection split across two header lines must still be detected as an upgrade request")
1339	}
1340	}
1341
1342	// activatorFunc adapts a plain function to the Activator interface, for
1343	// tests that want custom wake-counting behavior without a full fake type.
1344	type activatorFunc func(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool
1345
1346	func (f activatorFunc) ServeHTTP(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool {
1347	return f(w, r, rt, proxy)
1348	}
1349
1350	func (f activatorFunc) RetireExcept(keep map[string]bool) {
1351	// No-op for tests that don't need retirement tracking.
1352	}
1353
1354	// TestCacheBeforeWakeFullCycleOverPurgeSocket is the full cycle with the REAL
1355	// purge channel in it: the same cold-miss → cached-repeat → purge → fresh
1356	// sequence as TestCacheBeforeWakeFullCycle, except the purge arrives the way a
1357	// spawned instance actually sends it — a JSON POST /purge over the unix
1358	// control socket ListenPurge binds (the socket whose path the activator hands
1359	// the child as CARLOS_CONTROL_SOCKET). The sibling test below drives the
1360	// in-process cache API, which cannot catch a broken socket, encoding, or
1361	// host-scoping; this one proves the whole channel from "instance calls purge"
1362	// to "edge cache entry is gone."
1363	func TestCacheBeforeWakeFullCycleOverPurgeSocket(t *testing.T) {
1364	body := "version 1"
1365	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1366	w.Header().Set("Cache-Control", "max-age=60")
1367	w.WriteHeader(200)
1368	w.Write([]byte(body))
1369	}))
1370	t.Cleanup(backend.Close)
1371
1372	wakes := 0
1373	act := activatorFunc(func(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool {
1374	wakes++
1375	proxy.ServeHTTP(w, r)
1376	return true
1377	})
1378
1379	reg := openReg(t)
1380	if err := reg.Upsert(registry.Route{Host: "sock-cycle.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready", Hibernate: true}); err != nil {
1381	t.Fatalf("upsert: %v", err)
1382	}
1383	e, err := New(reg, act)
1384	if err != nil {
1385	t.Fatalf("New: %v", err)
1386	}
1387	cache := NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)
1388	e.EnableCache(cache)
1389
1390	// The control socket the activator would create for this instance and
1391	// hand it as CARLOS_CONTROL_SOCKET.
1392	controlSocket := filepath.Join(t.TempDir(), "instance.sock.control")
1393	pl, err := ListenPurge(controlSocket, "sock-cycle.example.com", cache)
1394	if err != nil {
1395	if errors.Is(err, syscall.EPERM) || strings.Contains(err.Error(), "operation not permitted") {
1396	t.Skipf("unix-socket listeners not permitted in this environment: %v", err)
1397	}
1398	t.Fatalf("ListenPurge: %v", err)
1399	}
1400	defer pl.Close()
1401
1402	get := func() string {
1403	rec := httptest.NewRecorder()
1404	e.ServeHTTP(rec, httptest.NewRequest("GET", "http://sock-cycle.example.com/event", nil))
1405	return rec.Body.String()
1406	}
1407
1408	if got := get(); got != "version 1" {
1409	t.Fatalf("first request body = %q, want %q", got, "version 1")
1410	}
1411	if wakes != 1 {
1412	t.Fatalf("expected exactly 1 wake for the cold request, got %d", wakes)
1413	}
1414	if got := get(); got != "version 1" || wakes != 1 {
1415	t.Fatalf("second request body = %q wakes = %d, want %q and no second wake", got, wakes, "version 1")
1416	}
1417
1418	// The write happens, and the instance purges its own page over the control
1419	// socket — exactly the request a real instance makes.
1420	body = "version 2"
1421	payload, _ := json.Marshal(purgeRequest{Path: "/event", Query: ""})
1422	resp, err := dialPurgeSocket(t, controlSocket).Post("http://unix/purge", "application/json", bytes.NewReader(payload))
1423	if err != nil {
1424	t.Fatalf("purge POST over control socket: %v", err)
1425	}
1426	resp.Body.Close()
1427	if resp.StatusCode != http.StatusOK {
1428	t.Fatalf("purge status = %d, want 200", resp.StatusCode)
1429	}
1430
1431	if got := get(); got != "version 2" {
1432	t.Fatalf("post-purge request body = %q, want %q — the purge never reached the cache through the socket", got, "version 2")
1433	}
1434	if wakes != 2 {
1435	t.Fatalf("expected the post-purge request to miss and wake once more, wakes = %d", wakes)
1436	}
1437	}
1438
1439	// TestCacheBeforeWakeFullCycle exercises the sequence the whole feature
1440	// exists for: a cold miss wakes the instance and populates the cache; a
1441	// repeat request hits the cache and does NOT wake the instance again; a
1442	// write's purge call makes the next request miss and see fresh content.
1443	func TestCacheBeforeWakeFullCycle(t *testing.T) {
1444	body := "version 1"
1445	backend := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1446	w.Header().Set("Cache-Control", "max-age=60")
1447	w.WriteHeader(200)
1448	w.Write([]byte(body))
1449	}))
1450	t.Cleanup(backend.Close)
1451
1452	wakes := 0
1453	act := activatorFunc(func(w http.ResponseWriter, r *http.Request, rt registry.Route, proxy http.Handler) bool {
1454	wakes++
1455	proxy.ServeHTTP(w, r)
1456	return true
1457	})
1458
1459	reg := openReg(t)
1460	if err := reg.Upsert(registry.Route{Host: "cycle.example.com", Kind: "instance", Addr: backend.Listener.Addr().String(), Status: "ready", Hibernate: true}); err != nil {
1461	t.Fatalf("upsert: %v", err)
1462	}
1463	e, err := New(reg, act)
1464	if err != nil {
1465	t.Fatalf("New: %v", err)
1466	}
1467	cache := NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)
1468	e.EnableCache(cache)
1469
1470	get := func() string {
1471	rec := httptest.NewRecorder()
1472	e.ServeHTTP(rec, httptest.NewRequest("GET", "http://cycle.example.com/event", nil))
1473	return rec.Body.String()
1474	}
1475
1476	if got := get(); got != "version 1" {
1477	t.Fatalf("first request body = %q, want %q", got, "version 1")
1478	}
1479	if wakes != 1 {
1480	t.Fatalf("expected exactly 1 wake for the cold request, got %d", wakes)
1481	}
1482
1483	if got := get(); got != "version 1" {
1484	t.Fatalf("second request body = %q, want %q", got, "version 1")
1485	}
1486	if wakes != 1 {
1487	t.Fatalf("expected the second request to hit cache without waking again, wakes = %d", wakes)
1488	}
1489
1490	// Simulate the write + purge an instance does after a real change.
1491	body = "version 2"
1492	cache.PurgeURL("cycle.example.com", "/event", "")
1493
1494	if got := get(); got != "version 2" {
1495	t.Fatalf("post-purge request body = %q, want %q", got, "version 2")
1496	}
1497	if wakes != 2 {
1498	t.Fatalf("expected the post-purge request to miss and wake once more, wakes = %d", wakes)
1499	}
1500	}
1501
1502	type mockResponseWriterFor1xx struct {
1503	header http.Header
1504	codes []int
1505	}
1506
1507	func (m *mockResponseWriterFor1xx) Header() http.Header {
1508	return m.header
1509	}
1510
1511	func (m *mockResponseWriterFor1xx) WriteHeader(status int) {
1512	// Stand-in for a real ResponseWriter. httptest.ResponseRecorder rejects
1513	// writes after a 1xx status (RFC 9110 forbids bodies on 1xx), but a real
1514	// ResponseWriter accepts the sequence 1xx then 2xx (103 Early Hints
1515	// followed by the final response status) and forwards both. This mock
1516	// accepts and records both.
1517	m.codes = append(m.codes, status)
1518	}
1519
1520	func (m *mockResponseWriterFor1xx) Write(p []byte) (int, error) {
1521	return len(p), nil
1522	}
1523
1524	func TestRecordingWriterIgnoresInformational(t *testing.T) {
1525	mock := &mockResponseWriterFor1xx{header: make(http.Header)}
1526
1527	rec := &recordingResponseWriter{
1528	ResponseWriter: mock,
1529	request: httptest.NewRequest("GET", "http://app.example/x", nil),
1530	cacheable: true,
1531	maxBody: 1 << 20,
1532	}
1533	rec.WriteHeader(http.StatusEarlyHints)
1534	rec.Header().Set("Cache-Control", "max-age=60")
1535	rec.WriteHeader(http.StatusOK)
1536	if _, err := rec.Write([]byte("final body")); err != nil {
1537	t.Fatal(err)
1538	}
1539	if rec.status != http.StatusOK {
1540	t.Fatalf("recorded status %d, want %d — informational must not latch", rec.status, http.StatusOK)
1541	}
1542	if !slices.Equal(mock.codes, []int{http.StatusEarlyHints, http.StatusOK}) {
1543	t.Fatalf("underlying writer saw %v, want [103 200] — the 103 must be forwarded, not swallowed", mock.codes)
1544	}
1545	e := &Edge{cache: NewCache(DefaultCacheMaxEntries, DefaultCacheMaxBodyBytes)}
1546	key := CacheKey{Host: "app.example", Method: "GET", Path: "/x"}
1547	e.maybeStore(key, rec)
1548	if _, ok := e.cache.get(key); !ok {
1549	t.Fatal("response written after a 103 was not cached")
1550	}
1551	}
1552
1553	// getHostHeader is getHost's sibling for tests that need to inspect a
1554	// response header rather than the body — the deploy watch's proof
1555	// (X-Carlos-Version) lives there, not in the body.
1556	func getHostHeader(t *testing.T, edgeURL, host string) *http.Response {
1557	t.Helper()
1558	req, err := http.NewRequest("GET", edgeURL+"/", nil)
1559	if err != nil {
1560	t.Fatalf("new request: %v", err)
1561	}
1562	req.Host = host
1563	resp, err := http.DefaultClient.Do(req)
1564	if err != nil {
1565	t.Fatalf("GET %s (host %s): %v", edgeURL, host, err)
1566	}
1567	defer resp.Body.Close()
1568	io.Copy(io.Discard, resp.Body)
1569	return resp
1570	}
1571
1572	func TestVersionHeaderOnVersionedRoute(t *testing.T) {
1573	backend, _ := hostEchoBackend(t, "hello from backend")
1574	addr := backend.Listener.Addr().String()
1575
1576	reg := openReg(t)
1577	if err := reg.Upsert(registry.Route{Host: "app.example.com", Kind: "service", Addr: addr, Status: "ready", Version: "abc1234"}); err != nil {
1578	t.Fatalf("upsert: %v", err)
1579	}
1580
1581	e, err := New(reg, nil)
1582	if err != nil {
1583	t.Fatalf("New: %v", err)
1584	}
1585	srv := httptest.NewServer(e)
1586	defer srv.Close()
1587
1588	resp := getHostHeader(t, srv.URL, "app.example.com")
1589	if got := resp.Header.Get("X-Carlos-Version"); got != "abc1234" {
1590	t.Fatalf("X-Carlos-Version = %q, want %q", got, "abc1234")
1591	}
1592	}
1593
1594	func TestNoVersionHeaderOnUnversionedRoute(t *testing.T) {
1595	backend, _ := hostEchoBackend(t, "hello from backend")
1596	addr := backend.Listener.Addr().String()
1597
1598	reg := openReg(t)
1599	if err := reg.Upsert(registry.Route{Host: "app.example.com", Kind: "service", Addr: addr, Status: "ready"}); err != nil {
1600	t.Fatalf("upsert: %v", err)
1601	}
1602
1603	e, err := New(reg, nil)
1604	if err != nil {
1605	t.Fatalf("New: %v", err)
1606	}
1607	srv := httptest.NewServer(e)
1608	defer srv.Close()
1609
1610	resp := getHostHeader(t, srv.URL, "app.example.com")
1611	if got, ok := resp.Header["X-Carlos-Version"]; ok {
1612	t.Fatalf("X-Carlos-Version present (%v), want absent for an unversioned route", got)
1613	}
1614	}
1615
1616	// ---------------------------------------------------------------------------
1617	// Multi-upstream addr routes (HA design Part II, issue #152)
1618	// ---------------------------------------------------------------------------
1619
1620	// testProbePath is the health path every multi-upstream test configures. It is
1621	// deliberately NOT "/": countingBackend ignores it, so a counter can never
1622	// confuse the prober's own traffic for proxied requests — a test that counted
1623	// probes would report "both upstreams were used" on a proxy that balanced
1624	// nothing at all.
1625	const testProbePath = "/healthz"
1626
1627	// countingBackend is an upstream that counts the PROXIED requests it served
1628	// and answers a fixed body. Probe requests (testProbePath) are answered 200
1629	// but never counted.
1630	func countingBackend(t *testing.T, body string) (string, *atomic.Int64) {
1631	t.Helper()
1632	hits := new(atomic.Int64)
1633	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1634	if r.URL.Path == testProbePath {
1635	w.WriteHeader(http.StatusOK)
1636	return
1637	}
1638	hits.Add(1)
1639	io.WriteString(w, body)
1640	}))
1641	t.Cleanup(ts.Close)
1642	return ts.Listener.Addr().String(), hits
1643	}
1644
1645	// deadAddr reserves a real loopback port and closes the listener immediately,
1646	// yielding an address that is well-formed and guaranteed to REFUSE
1647	// connections. That refusal is instant, so a test asserting the dial-failure
1648	// path never waits out a timeout — and, unlike a made-up port, it cannot
1649	// collide with something else the machine is running.
1650	func deadAddr(t *testing.T) string {
1651	t.Helper()
1652	ln, err := net.Listen("tcp", "127.0.0.1:0")
1653	if err != nil {
1654	t.Fatalf("listen: %v", err)
1655	}
1656	addr := ln.Addr().String()
1657	if err := ln.Close(); err != nil {
1658	t.Fatalf("close reserved listener: %v", err)
1659	}
1660	return addr
1661	}
1662
1663	// healthJSON marshals a probe config the way provision does when it mints a
1664	// route, so the tests exercise the same column bytes Reload parses in
1665	// production.
1666	func healthJSON(t *testing.T, h provision.Health) string {
1667	t.Helper()
1668	b, err := json.Marshal(h)
1669	if err != nil {
1670	t.Fatalf("marshal health: %v", err)
1671	}
1672	return string(b)
1673	}
1674
1675	// frozenHealth is a probe config whose cadence cannot fire within a test: one
1676	// tick runs immediately on Sync (hostProber.run), then nothing for a minute,
1677	// and fail_after is far beyond what a single tick could reach. Any live-set
1678	// change a test observes under it therefore came from the REQUEST path's
1679	// dial-failure fast path (proberSet.Eject), never from a probe.
1680	func frozenHealth() provision.Health {
1681	return provision.Health{Path: testProbePath, Status: 200, IntervalMS: 60000, FailAfter: 1000, RiseAfter: 1000}
1682	}
1683
1684	func TestMultiUpstreamBalancesAcrossLiveBackends(t *testing.T) {
1685	addrA, hitsA := countingBackend(t, "A")
1686	addrB, hitsB := countingBackend(t, "B")
1687
1688	reg := openReg(t)
1689	if err := reg.Upsert(registry.Route{
1690	Host: "ha.example.com", Kind: "service", Status: "ready",
1691	Addrs: addrA + "," + addrB,
1692	Health: healthJSON(t, frozenHealth()),
1693	}); err != nil {
1694	t.Fatalf("upsert: %v", err)
1695	}
1696
1697	e, err := New(reg, nil)
1698	if err != nil {
1699	t.Fatalf("New: %v", err)
1700	}
1701	defer e.probers.Stop()
1702	srv := httptest.NewServer(e)
1703	defer srv.Close()
1704
1705	for i := 0; i < 20; i++ {
1706	code, body := getHost(t, srv.URL, "ha.example.com")
1707	if code != http.StatusOK {
1708	t.Fatalf("request %d: status = %d, want 200", i, code)
1709	}
1710	if body != "A" && body != "B" {
1711	t.Fatalf("request %d: body = %q, want one of the two upstreams", i, body)
1712	}
1713	}
1714	// Exactly even, not merely "both were used". The cursor is per route and
1715	// starts at zero, keep-alives are off so every request dials, and both
1716	// upstreams stay live throughout — so 10/10 is the only correct answer
1717	// here, and a loose "> 0" assertion would sit quietly through a 19/1 split
1718	// that means the rotation has broken.
1719	if hitsA.Load() != 10 || hitsB.Load() != 10 {
1720	t.Fatalf("upstream hits = A:%d B:%d, want 10/10 (strict round-robin over the live set)",
1721	hitsA.Load(), hitsB.Load())
1722	}
1723	}
1724
1725	func TestDialFailureRetriesOtherUpstreamNoUserError(t *testing.T) {
1726	dead := deadAddr(t)
1727	addrA, hitsA := countingBackend(t, "A")
1728
1729	reg := openReg(t)
1730	// Dead upstream FIRST in the list, so the very first request must retry
1731	// past it inside the same request rather than answering an error.
1732	if err := reg.Upsert(registry.Route{
1733	Host: "ha.example.com", Kind: "service", Status: "ready",
1734	Addrs: dead + "," + addrA,
1735	Health: healthJSON(t, frozenHealth()),
1736	}); err != nil {
1737	t.Fatalf("upsert: %v", err)
1738	}
1739
1740	e, err := New(reg, nil)
1741	if err != nil {
1742	t.Fatalf("New: %v", err)
1743	}
1744	defer e.probers.Stop()
1745	srv := httptest.NewServer(e)
1746	defer srv.Close()
1747
1748	for i := 0; i < 10; i++ {
1749	code, body := getHost(t, srv.URL, "ha.example.com")
1750	if code != http.StatusOK {
1751	t.Fatalf("request %d: status = %d, want 200 — a dead upstream must cost a retry, not a user-visible error", i, code)
1752	}
1753	if body != "A" {
1754	t.Fatalf("request %d: body = %q, want %q (the live upstream)", i, body, "A")
1755	}
1756	}
1757	if got := hitsA.Load(); got != 10 {
1758	t.Fatalf("live upstream served %d requests, want 10", got)
1759	}
1760	// The fast path: frozenHealth's cadence cannot have ejected anything, so
1761	// this can only be the in-request dial failure.
1762	if live := e.probers.Live("ha.example.com"); !slices.Equal(live, []string{addrA}) {
1763	t.Fatalf("live set = %v, want [%s] — a failed dial must eject %s immediately", live, addrA, dead)
1764	}
1765	}
1766
1767	func TestAllUpstreamsDownAnswers503WithRetryAfter(t *testing.T) {
1768	dead1, dead2 := deadAddr(t), deadAddr(t)
1769
1770	reg := openReg(t)
1771	if err := reg.Upsert(registry.Route{
1772	Host: "ha.example.com", Kind: "service", Status: "ready",
1773	Addrs: dead1 + "," + dead2,
1774	Health: healthJSON(t, frozenHealth()),
1775	}); err != nil {
1776	t.Fatalf("upsert: %v", err)
1777	}
1778
1779	e, err := New(reg, nil)
1780	if err != nil {
1781	t.Fatalf("New: %v", err)
1782	}
1783	defer e.probers.Stop()
1784	srv := httptest.NewServer(e)
1785	defer srv.Close()
1786
1787	resp := getHostHeader(t, srv.URL, "ha.example.com")
1788	if resp.StatusCode != http.StatusServiceUnavailable {
1789	t.Fatalf("status = %d, want 503 (a dark upstream set is 'come back shortly', not the stdlib's bare 502)", resp.StatusCode)
1790	}
1791	if got := resp.Header.Get("Retry-After"); got != "2" {
1792	t.Fatalf("Retry-After = %q, want %q", got, "2")
1793	}
1794	// A second request must answer the same way rather than hanging or
1795	// escalating: every upstream is ejected by now, so the live set is empty
1796	// and the transport never dials at all.
1797	resp2 := getHostHeader(t, srv.URL, "ha.example.com")
1798	if resp2.StatusCode != http.StatusServiceUnavailable || resp2.Header.Get("Retry-After") != "2" {
1799	t.Fatalf("second request = %d Retry-After %q, want 503/2 with an empty live set",
1800	resp2.StatusCode, resp2.Header.Get("Retry-After"))
1801	}
1802	if live := e.probers.Live("ha.example.com"); len(live) != 0 {
1803	t.Fatalf("live set = %v, want empty after both upstreams refused", live)
1804	}
1805	}
1806
1807	func TestSingleAddrWithHealthGets503Semantics(t *testing.T) {
1808	dead := deadAddr(t)
1809
1810	reg := openReg(t)
1811	// One upstream, carried in the LEGACY Addr column, but with health
1812	// configured: probing is what buys the 503 semantics, not the count.
1813	if err := reg.Upsert(registry.Route{
1814	Host: "solo.example.com", Kind: "service", Status: "ready",
1815	Addr: dead,
1816	Health: healthJSON(t, frozenHealth()),
1817	}); err != nil {
1818	t.Fatalf("upsert: %v", err)
1819	}
1820
1821	e, err := New(reg, nil)
1822	if err != nil {
1823	t.Fatalf("New: %v", err)
1824	}
1825	defer e.probers.Stop()
1826	srv := httptest.NewServer(e)
1827	defer srv.Close()
1828
1829	resp := getHostHeader(t, srv.URL, "solo.example.com")
1830	if resp.StatusCode != http.StatusServiceUnavailable {
1831	t.Fatalf("status = %d, want 503 for a probed single upstream", resp.StatusCode)
1832	}
1833	if got := resp.Header.Get("Retry-After"); got != "2" {
1834	t.Fatalf("Retry-After = %q, want %q", got, "2")
1835	}
1836	// A probed single-addr route is on the dashboard like any other.
1837	if st, ok := e.probers.State()["solo.example.com"]; !ok || len(st) != 1 || st[0].Addr != dead {
1838	t.Fatalf("prober state = %v, want one entry for %s", st, dead)
1839	}
1840	}
1841
1842	// TestUpstreamStatesExportsProbeTable pins Task 6's accessor: the box status
1843	// snapshot's only door onto the probe table is Edge.UpstreamStates, which must
1844	// report exactly what the unexported proberSet holds — same hosts, same addrs,
1845	// same order — and must be safe to call on an edge with nothing probed (a
1846	// pure-proxy `carlos edge`, which is most of them), where it reports empty
1847	// rather than nil-panicking or inventing a host.
1848	func TestUpstreamStatesExportsProbeTable(t *testing.T) {
1849	addrA, _ := countingBackend(t, "A")
1850	dead := deadAddr(t)
1851
1852	reg := openReg(t)
1853	if err := reg.Upsert(registry.Route{
1854	Host: "ha.example.com", Kind: "service", Status: "ready",
1855	Addrs: addrA + "," + dead,
1856	Health: healthJSON(t, frozenHealth()),
1857	}); err != nil {
1858	t.Fatalf("upsert: %v", err)
1859	}
1860	// A second, UNPROBED route: no health config, so it holds no prober and
1861	// must not appear in the exported table at all.
1862	if err := reg.Upsert(registry.Route{
1863	Host: "plain.example.com", Kind: "service", Status: "ready", Addr: addrA,
1864	}); err != nil {
1865	t.Fatalf("upsert: %v", err)
1866	}
1867
1868	e, err := New(reg, nil)
1869	if err != nil {
1870	t.Fatalf("New: %v", err)
1871	}
1872	defer e.probers.Stop()
1873
1874	got := e.UpstreamStates()
1875	if len(got) != 1 {
1876	t.Fatalf("UpstreamStates() = %+v, want exactly the one probed host", got)
1877	}
1878	states := got["ha.example.com"]
1879	if len(states) != 2 || states[0].Addr != addrA || states[1].Addr != dead {
1880	t.Fatalf("states = %+v, want [%s %s] in the route's addr order", states, addrA, dead)
1881	}
1882	// Optimistic start (prober.go rule 1) is what the snapshot reports for a
1883	// freshly synced host — frozenHealth cannot have ticked a transition in.
1884	for _, st := range states {
1885	if !st.Up || st.Since.IsZero() {
1886	t.Errorf("state %+v: a freshly synced upstream must be up with a Since stamp", st)
1887	}
1888	}
1889
1890	// The exported view is a COPY: mutating it must not reach the live table
1891	// the request path picks from.
1892	states[0].Up = false
1893	if again := e.UpstreamStates()["ha.example.com"]; !again[0].Up {
1894	t.Error("UpstreamStates() handed out a view of live state — the snapshot must never be able to eject an upstream")
1895	}
1896
1897	// An edge with nothing probed answers empty, not nil-panic.
1898	plainReg := openReg(t)
1899	if err := plainReg.Upsert(registry.Route{Host: "plain.example.com", Kind: "service", Status: "ready", Addr: addrA}); err != nil {
1900	t.Fatalf("upsert: %v", err)
1901	}
1902	plain, err := New(plainReg, nil)
1903	if err != nil {
1904	t.Fatalf("New (unprobed): %v", err)
1905	}
1906	defer plain.probers.Stop()
1907	if st := plain.UpstreamStates(); len(st) != 0 {
1908	t.Errorf("UpstreamStates() on an unprobed edge = %+v, want empty", st)
1909	}
1910	}
1911
1912	func TestLegacySingleAddrRouteUnchanged(t *testing.T) {
1913	dead := deadAddr(t)
1914
1915	reg := openReg(t)
1916	if err := reg.Upsert(registry.Route{
1917	Host: "legacy.example.com", Kind: "service", Status: "ready", Addr: dead,
1918	}); err != nil {
1919	t.Fatalf("upsert: %v", err)
1920	}
1921
1922	e, err := New(reg, nil)
1923	if err != nil {
1924	t.Fatalf("New: %v", err)
1925	}
1926	defer e.probers.Stop()
1927	srv := httptest.NewServer(e)
1928	defer srv.Close()
1929
1930	resp := getHostHeader(t, srv.URL, "legacy.example.com")
1931	if resp.StatusCode != http.StatusBadGateway {
1932	t.Fatalf("status = %d, want 502 — an UNPROBED route must keep its pre-multi-upstream behavior byte for byte", resp.StatusCode)
1933	}
1934	if got := resp.Header.Get("Retry-After"); got != "" {
1935	t.Fatalf("Retry-After = %q, want absent on the legacy path", got)
1936	}
1937	if st := e.probers.State(); len(st) != 0 {
1938	t.Fatalf("prober state = %v, want empty — a route with no health config is not probed", st)
1939	}
1940	}
1941
1942	func TestReloadSwapsUpstreamSet(t *testing.T) {
1943	addrA, _ := countingBackend(t, "A")
1944	addrB, _ := countingBackend(t, "B")
1945	health := healthJSON(t, frozenHealth())
1946
1947	reg := openReg(t)
1948	if err := reg.Upsert(registry.Route{
1949	Host: "ha.example.com", Kind: "service", Status: "ready",
1950	Addrs: addrA, Health: health,
1951	}); err != nil {
1952	t.Fatalf("upsert: %v", err)
1953	}
1954
1955	e, err := New(reg, nil)
1956	if err != nil {
1957	t.Fatalf("New: %v", err)
1958	}
1959	defer e.probers.Stop()
1960	srv := httptest.NewServer(e)
1961	defer srv.Close()
1962
1963	if code, body := getHost(t, srv.URL, "ha.example.com"); code != http.StatusOK || body != "A" {
1964	t.Fatalf("pre-reload = %d %q, want 200 %q", code, body, "A")
1965	}
1966
1967	// Write via a SECOND handle, the cross-process shape a reconcile pass has.
1968	reg2, err := registry.Open(reg.Path())
1969	if err != nil {
1970	t.Fatalf("open second handle: %v", err)
1971	}
1972	if err := reg2.SetUpstreams("ha.example.com", "", addrB, health); err != nil {
1973	t.Fatalf("SetUpstreams: %v", err)
1974	}
1975	if err := e.Reload(); err != nil {
1976	t.Fatalf("Reload: %v", err)
1977	}
1978
1979	if code, body := getHost(t, srv.URL, "ha.example.com"); code != http.StatusOK || body != "B" {
1980	t.Fatalf("post-reload = %d %q, want 200 %q (the new upstream set must be served)", code, body, "B")
1981	}
1982	st := e.probers.State()["ha.example.com"]
1983	if len(st) != 1 || st[0].Addr != addrB {
1984	t.Fatalf("prober state = %v, want exactly one entry for %s — the probe specs must follow the route", st, addrB)
1985	}
1986	}
1987
1988	// upgradeEchoBackend answers a protocol-switch handshake the way a WebSocket
1989	// server does — 101, then raw bytes on the hijacked connection — prefixing
1990	// whatever it reads with tag so a test can tell WHICH upstream served it.
1991	func upgradeEchoBackend(t *testing.T, tag string) string {
1992	t.Helper()
1993	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
1994	if r.URL.Path == testProbePath {
1995	w.WriteHeader(http.StatusOK)
1996	return
1997	}
1998	hj, ok := w.(http.Hijacker)
1999	if !ok {
2000	http.Error(w, "not hijackable", http.StatusInternalServerError)
2001	return
2002	}
2003	conn, buf, err := hj.Hijack()
2004	if err != nil {
2005	return
2006	}
2007	defer conn.Close()
2008	buf.WriteString("HTTP/1.1 101 Switching Protocols\r\nUpgrade: websocket\r\nConnection: Upgrade\r\n\r\n")
2009	if err := buf.Flush(); err != nil {
2010	return
2011	}
2012	line, err := buf.ReadString('\n')
2013	if err != nil {
2014	return
2015	}
2016	buf.WriteString(tag + ":" + line)
2017	buf.Flush()
2018	}))
2019	t.Cleanup(ts.Close)
2020	return ts.Listener.Addr().String()
2021	}
2022
2023	// TestMultiUpstreamServesWebSocketUpgrade guards the one thing multiProxy's
2024	// upstream keep-alive suppression could plausibly have broken. A transport
2025	// with DisableKeepAlives normally adds `Connection: close` to its outbound
2026	// request, which would poison a protocol-switch handshake — the stdlib
2027	// exempts upgrade requests from that (transport.go's isProtocolSwitchHeader
2028	// check), and this test is what keeps that exemption load-bearing rather than
2029	// assumed.
2030	func TestMultiUpstreamServesWebSocketUpgrade(t *testing.T) {
2031	addr := upgradeEchoBackend(t, "ws")
2032
2033	reg := openReg(t)
2034	if err := reg.Upsert(registry.Route{
2035	Host: "ws.example.com", Kind: "service", Status: "ready",
2036	Addrs: addr, Health: healthJSON(t, frozenHealth()),
2037	}); err != nil {
2038	t.Fatalf("upsert: %v", err)
2039	}
2040	e, err := New(reg, nil)
2041	if err != nil {
2042	t.Fatalf("New: %v", err)
2043	}
2044	defer e.probers.Stop()
2045	srv := httptest.NewServer(e)
2046	defer srv.Close()
2047
2048	// Pin that this route really took the multiProxy arm: only a probed route
2049	// has a prober, so without this the test would still pass if the switch
2050	// had quietly built a plain tcpProxy — and it is tcpProxy's transport,
2051	// not multiProxy's keep-alive-free one, that the upgrade path is being
2052	// checked against.
2053	if live := e.probers.Live("ws.example.com"); len(live) != 1 {
2054	t.Fatalf("live set = %v, want one probed upstream — this test must exercise multiProxy, not tcpProxy", live)
2055	}
2056
2057	c, err := net.Dial("tcp", strings.TrimPrefix(srv.URL, "http://"))
2058	if err != nil {
2059	t.Fatalf("dial edge: %v", err)
2060	}
2061	defer c.Close()
2062	if err := c.SetDeadline(time.Now().Add(10 * time.Second)); err != nil {
2063	t.Fatalf("set deadline: %v", err)
2064	}
2065	if _, err := fmt.Fprint(c, "GET /socket HTTP/1.1\r\nHost: ws.example.com\r\nConnection: Upgrade\r\nUpgrade: websocket\r\n\r\n"); err != nil {
2066	t.Fatalf("write handshake: %v", err)
2067	}
2068	br := bufio.NewReader(c)
2069	resp, err := http.ReadResponse(br, nil)
2070	if err != nil {
2071	t.Fatalf("read handshake response: %v", err)
2072	}
2073	if resp.StatusCode != http.StatusSwitchingProtocols {
2074	t.Fatalf("handshake status = %d, want 101 — an upgrade over a multi-upstream route must switch protocols", resp.StatusCode)
2075	}
2076	if _, err := fmt.Fprint(c, "ping\n"); err != nil {
2077	t.Fatalf("write post-upgrade frame: %v", err)
2078	}
2079	line, err := br.ReadString('\n')
2080	if err != nil {
2081	t.Fatalf("read post-upgrade frame: %v", err)
2082	}
2083	if line != "ws:ping\n" {
2084	t.Fatalf("post-upgrade echo = %q, want %q — bytes must flow both ways after the switch", line, "ws:ping\n")
2085	}
2086	}
2087
2088	// TestUnparseableHealthServesUnprobed pins the parse-failure contract: a
2089	// health column the edge cannot read costs the route its PROBING, never its
2090	// SERVICE. The alternative — skipping the route, or building a proxy over an
2091	// empty live set — turns one bad character in a config column into a dead
2092	// host, which is the one outcome this branch exists to prevent.
2093	func TestUnparseableHealthServesUnprobed(t *testing.T) {
2094	addr, hits := countingBackend(t, "served anyway")
2095
2096	reg := openReg(t)
2097	if err := reg.Upsert(registry.Route{
2098	Host: "broken.example.com", Kind: "service", Status: "ready",
2099	Addrs: addr, Health: "{not json",
2100	}); err != nil {
2101	t.Fatalf("upsert: %v", err)
2102	}
2103
2104	e, err := New(reg, nil)
2105	if err != nil {
2106	t.Fatalf("New: %v", err)
2107	}
2108	defer e.probers.Stop()
2109	srv := httptest.NewServer(e)
2110	defer srv.Close()
2111
2112	code, body := getHost(t, srv.URL, "broken.example.com")
2113	if code != http.StatusOK || body != "served anyway" {
2114	t.Fatalf("= %d %q, want 200 %q — an unreadable health config must not take the host off the air",
2115	code, body, "served anyway")
2116	}
2117	if got := hits.Load(); got != 1 {
2118	t.Fatalf("backend served %d requests, want 1", got)
2119	}
2120	if st := e.probers.State(); len(st) != 0 {
2121	t.Fatalf("prober state = %v, want empty — a health config that did not parse cannot become a probing job", st)
2122	}
2123	}
2124
2125	// TestMultiAddrWithoutHealthServesFirstUnprobed covers the row shape nothing
2126	// mints but the switch must still answer for: several addrs, no health
2127	// config. Before multi-upstream routing such a row was served by NOTHING (the
2128	// old arm gated on the legacy Addr column, which this shape leaves empty), so
2129	// the behavior asserted here is new by construction and needs pinning:
2130	// upstream ONE is served, unprobed, with the legacy 502 semantics. Balancing
2131	// is deliberately not offered — without a probe there is no live set, and
2132	// rotating onto upstreams nothing is checking would be worse than using one.
2133	func TestMultiAddrWithoutHealthServesFirstUnprobed(t *testing.T) {
2134	first, firstHits := countingBackend(t, "first")
2135	second, secondHits := countingBackend(t, "second")
2136	dead := deadAddr(t)
2137
2138	reg := openReg(t)
2139	if err := reg.Upsert(registry.Route{
2140	Host: "unprobed.example.com", Kind: "service", Status: "ready",
2141	Addrs: first + "," + second,
2142	}); err != nil {
2143	t.Fatalf("upsert: %v", err)
2144	}
2145	// A second route of the same shape whose first upstream is dead, so the
2146	// failure semantics can be asserted alongside the success ones.
2147	if err := reg.Upsert(registry.Route{
2148	Host: "unprobed-dead.example.com", Kind: "service", Status: "ready",
2149	Addrs: dead + "," + second,
2150	}); err != nil {
2151	t.Fatalf("upsert: %v", err)
2152	}
2153
2154	e, err := New(reg, nil)
2155	if err != nil {
2156	t.Fatalf("New: %v", err)
2157	}
2158	defer e.probers.Stop()
2159	srv := httptest.NewServer(e)
2160	defer srv.Close()
2161
2162	for i := 0; i < 6; i++ {
2163	code, body := getHost(t, srv.URL, "unprobed.example.com")
2164	if code != http.StatusOK || body != "first" {
2165	t.Fatalf("request %d = %d %q, want 200 %q (addrs[0], unprobed)", i, code, body, "first")
2166	}
2167	}
2168	if got := firstHits.Load(); got != 6 {
2169	t.Fatalf("first upstream served %d requests, want all 6", got)
2170	}
2171	if got := secondHits.Load(); got != 0 {
2172	t.Fatalf("second upstream served %d requests, want 0 — an unprobed route must not balance onto upstreams nothing is checking", got)
2173	}
2174
2175	// Dead first upstream: the LEGACY failure shape, not the probed one. No
2176	// live set exists to fail away from, so there is nothing to promise a
2177	// client about coming back.
2178	resp := getHostHeader(t, srv.URL, "unprobed-dead.example.com")
2179	if resp.StatusCode != http.StatusBadGateway {
2180	t.Fatalf("dead first upstream = %d, want 502 (the unprobed path keeps the stdlib behavior)", resp.StatusCode)
2181	}
2182	if got := resp.Header.Get("Retry-After"); got != "" {
2183	t.Fatalf("Retry-After = %q, want absent — 503 semantics are bought by probing, not by the addr count", got)
2184	}
2185	if got := secondHits.Load(); got != 0 {
2186	t.Fatalf("second upstream served %d requests, want 0 — the unprobed path must not retry onto a sibling upstream", got)
2187	}
2188	if st := e.probers.State(); len(st) != 0 {
2189	t.Fatalf("prober state = %v, want empty — neither route carries a health config", st)
2190	}
2191	}
2192
2193	// TestServeHTTPMetersRequestsAndBytesExcludingStatus drives a full Edge over
2194	// HTTP (mirroring TestRoutesHostToTCPUpstream) with EnableUsage armed, and
2195	// proves the single wrap site in ServeHTTP counts every plain-path request
2196	// while a Kind=="status" route (the platform's own status page — not
2197	// billable, per the placement rules) never shows up in Drain().
2198	func TestServeHTTPMetersRequestsAndBytesExcludingStatus(t *testing.T) {
2199	body := "metered response body"
2200	backend, _ := hostEchoBackend(t, body)
2201	addr := backend.Listener.Addr().String()
2202
2203	reg := openReg(t)
2204	if err := reg.Upsert(registry.Route{Host: "metered.example.com", Kind: "service", Addr: addr, Account: "bab", App: "hello", Status: "ready"}); err != nil {
2205	t.Fatalf("upsert metered route: %v", err)
2206	}
2207	if err := reg.Upsert(registry.Route{Host: "status.example.com", Kind: "status", Status: "ready"}); err != nil {
2208	t.Fatalf("upsert status route: %v", err)
2209	}
2210
2211	e, err := New(reg, nil)
2212	if err != nil {
2213	t.Fatalf("New: %v", err)
2214	}
2215	// Static/status serving needs a store wired in for the status route to
2216	// answer at all (see testEdgeWithStore); an empty pubkey is fine since
2217	// the status class never verifies signatures.
2218	store := release.NewDirStore(filepath.Join(t.TempDir(), "store"))
2219	_, pubB64, err := release.GenerateKey()
2220	if err != nil {
2221	t.Fatalf("generate key: %v", err)
2222	}
2223	pub, err := release.ParsePublicKey(pubB64, "test")
2224	if err != nil {
2225	t.Fatalf("parse public key: %v", err)
2226	}
2227	e.EnableStatic(store, pub, filepath.Join(t.TempDir(), "cache"))
2228	store.Put(context.Background(), "status/index.html", []byte("<html>status</html>"), "")
2229	if err := e.Reload(); err != nil {
2230	t.Fatalf("Reload: %v", err)
2231	}
2232
2233	meter := NewUsageMeter()
2234	e.EnableUsage(meter)
2235
2236	srv := httptest.NewServer(e)
2237	defer srv.Close()
2238
2239	for i := 0; i < 2; i++ {
2240	code, got := getHost(t, srv.URL, "metered.example.com")
2241	if code != http.StatusOK || got != body {
2242	t.Fatalf("request %d: %d %q, want 200 %q", i, code, got, body)
2243	}
2244	}
2245	if code, _ := getHost(t, srv.URL, "status.example.com"); code != http.StatusOK {
2246	t.Fatalf("status route: %d, want 200", code)
2247	}
2248
2249	rows := meter.Drain()
2250	row := rows["metered.example.com"]
2251	wantBytes := int64(2 * len(body))
2252	if row.Requests != 2 || row.BytesOut != wantBytes {
2253	t.Fatalf("metered.example.com row = %+v, want Requests=2 BytesOut=%d", row, wantBytes)
2254	}
2255	if row.Account != "bab" || row.App != "hello" {
2256	t.Fatalf("metered.example.com row = %+v, want Account=bab App=hello", row)
2257	}
2258	if _, ok := rows["status.example.com"]; ok {
2259	t.Fatalf("status route must not be metered, got %+v", rows["status.example.com"])
2260	}
2261	}
2262
2263	// TestSidecarRouteIsNeverServable is the structural guarantee behind the
2264	// sidecar member kind: a sidecar is a process that serves nothing and that
2265	// nothing may dial, and the edge must refuse it on KIND rather than on the
2266	// accident of it carrying no backend columns.
2267	//
2268	// The row below is adversarial on purpose. It carries a Socket AND an Addr
2269	// AND a Tunnel AND a fleet placement (Region + FleetBox), on an edge with a
2270	// tunnel hub armed and the fleet registered — every column and every
2271	// precondition that any arm of Reload's dispatch switch looks at, all at
2272	// once. Eleven's sidecars are fleet-placed, so the fleet arm (which checks no
2273	// kind at all) is reached by such a row on shape alone. If this test ever
2274	// fails, a sidecar is reachable from the internet.
2275	func TestSidecarRouteIsNeverServable(t *testing.T) {
2276	const host = "bot.eleven.bdm.sidecar.internal"
2277
2278	reg := openReg(t)
2279	if err := reg.UpsertFleet(registry.Fleet{Name: "eleven", Account: "amadan", Bucket: "eleven-data"}); err != nil {
2280	t.Fatalf("UpsertFleet: %v", err)
2281	}
2282	if err := reg.AddFleetBox(registry.FleetBox{Fleet: "eleven", Label: "box1", TokenHash: sha256Hex("tok"), Status: "active"}); err != nil {
2283	t.Fatalf("AddFleetBox: %v", err)
2284	}
2285	if err := reg.Upsert(registry.Route{
2286	Host: host,
2287	Kind: "sidecar",
2288	Socket: filepath.Join(t.TempDir(), "whatever.sock"),
2289	Addr: "127.0.0.1:9",
2290	Tunnel: "eleven/box1",
2291	Region: "fleet/eleven",
2292	FleetBox: "box1",
2293	Status: "ready",
2294	App: "eleven",
2295	Account: "bdm",
2296	Platform: true,
2297	}); err != nil {
2298	t.Fatalf("upsert sidecar route: %v", err)
2299	}
2300
2301	e, err := New(reg, nil)
2302	if err != nil {
2303	t.Fatalf("New: %v", err)
2304	}
2305	// Arm the hub, so the Tunnel and fleet arms of the dispatch switch are
2306	// live rather than skipped by their e.tunnels == nil guard.
2307	e.EnableTunnels(NewTunnelHub("tunnel.example.com", nil))
2308	if err := e.Reload(); err != nil {
2309	t.Fatalf("Reload: %v", err)
2310	}
2311
2312	if _, ok := e.current()[host]; ok {
2313	t.Fatalf("a sidecar row entered the routing map — it is reachable from the internet")
2314	}
2315	// Allowed is the autocert HostPolicy: a sidecar host must never cost an
2316	// ACME issuance either.
2317	if e.Allowed(host) {
2318	t.Fatalf("Allowed(%s) = true — a certificate must never be requested for a sidecar host", host)
2319	}
2320
2321	srv := httptest.NewServer(e)
2322	defer srv.Close()
2323	if code, _ := getHost(t, srv.URL, host); code != http.StatusNotFound {
2324	t.Fatalf("GET %s = %d, want 404 — a sidecar must be unreachable", host, code)
2325	}
2326	}
2327