amadan
static-cache-control
Squashed from static-cache-control into main.
2 files changed,
+897
−2
internal/edge/static.go+301 −2internal/edge/static_cache_test.go+596 −0
CI cancelled — superseded by a newer push — run details
diff --git a/internal/edge/static.go b/internal/edge/static.go| index ea43c68ba..247ca200d 100644 |
| --- a/internal/edge/static.go |
| +++ b/internal/edge/static.go |
| @@ -17,16 +17,20 @@ import ( |
| "crypto/ed25519" |
| "crypto/sha256" |
| "encoding/hex" |
| + "errors" |
| "fmt" |
| "io" |
| "io/fs" |
| "log" |
| + "mime" |
| "net/http" |
| "os" |
| "path" |
| "path/filepath" |
| "sort" |
| + "strings" |
| "sync" |
| + "sync/atomic" |
| "time" |
| "github.com/carlosframework/platform/internal/apperr" |
| @@ -163,6 +167,18 @@ func (s *staticSite) ServeHTTP(w http.ResponseWriter, r *http.Request) { |
| // Set before any body is written, so it lands on the tree's own 404s too |
| // (the handler serving that tree IS a version). |
| w.Header().Set("X-Carlos-Version", version) |
| + // A page out of the tree is revalidated before it is reused, and so is |
| + // the tree's own 404. With Last-Modified alone a browser GUESSES a |
| + // freshness lifetime — typically a tenth of the file's age — and reuses |
| + // its copy without asking, so a site last deployed a month ago kept |
| + // showing the old page for days after the next deploy (discussion #195). |
| + // no-cache is "store it, but ask first"; the ETag serveTree sets is what |
| + // makes asking cost a 304. It also keeps these responses out of the |
| + // edge's own shared cache, which admits only a positive lifetime — as |
| + // they always were, having named none. (net/http strips the header again |
| + // from the errors it writes itself, a 416 or a 400: neither is a status a |
| + // browser caches on a guess.) |
| + w.Header().Set("Cache-Control", "no-cache") |
| s.serveTree(dir, w, r) |
| } |
| @@ -363,7 +379,26 @@ func extractSiteTar(data []byte, dst string) error { |
| // serveTree answers one request from an extracted version dir: |
| // http.FileServer semantics with index.html for directories and a clean 404 |
| // (never a directory listing) when neither the file nor an index exists. |
| +// |
| +// It serves the file itself rather than through http.ServeFile, because |
| +// ServeFile offers the file's mtime as Last-Modified and answers |
| +// If-Modified-Since from it — and an extracted file's mtime is when THIS box |
| +// unpacked the tree, which is not a fact about the content. It differs |
| +// between edges serving the same release, and it runs backwards on a |
| +// rollback: the tree rolled back to was unpacked before the one the browser |
| +// holds, so "modified since?" answers no and the browser keeps the build that |
| +// was withdrawn. No date is offered, so there is none to compare. |
| +// |
| +// The validator is the file's content (tagCache). What ServeFile did besides |
| +// is kept by hand: the refusal of a ".." element, and the redirect off a |
| +// trailing /index.html onto its directory. |
| func (s *staticSite) serveTree(dir string, w http.ResponseWriter, r *http.Request) { |
| + if containsDotDot(r.URL.Path) { |
| + // The Clean below would resolve it harmlessly inside the tree; this |
| + // keeps the answer ServeFile always gave such a path. |
| + http.Error(w, "invalid URL path", http.StatusBadRequest) |
| + return |
| + } |
| // Clean rooted at "/" so no request path can climb out of the tree. |
| p := path.Clean("/" + r.URL.Path) |
| fp := filepath.Join(dir, filepath.FromSlash(p)) |
| @@ -376,6 +411,270 @@ func (s *staticSite) serveTree(dir string, w http.ResponseWriter, r *http.Reques |
| http.NotFound(w, r) |
| return |
| } |
| - // ServeFile brings Content-Type, ranges, If-Modified-Since for free. |
| - http.ServeFile(w, r, fp) |
| + if strings.HasSuffix(r.URL.Path, "/index.html") { |
| + // Relative, as ServeFile wrote it, so it survives any prefix the |
| + // request arrived under. |
| + loc := "./" |
| + if q := r.URL.RawQuery; q != "" { |
| + loc += "?" + q |
| + } |
| + w.Header().Set("Location", loc) |
| + w.WriteHeader(http.StatusMovedPermanently) |
| + return |
| + } |
| + f, err := os.Open(fp) |
| + if err != nil { |
| + // ServeFile's mapping, kept: a file that vanished since the Stat is |
| + // a 404, but one that is there and will not open is not a missing |
| + // page, and must not read as one in an access log. |
| + switch { |
| + case errors.Is(err, fs.ErrNotExist): |
| + http.NotFound(w, r) |
| + case errors.Is(err, fs.ErrPermission): |
| + http.Error(w, "403 Forbidden", http.StatusForbidden) |
| + default: |
| + log.Printf("edge: static %s/%s: open %s: %v", s.app, s.channel, p, err) |
| + http.Error(w, "500 Internal Server Error", http.StatusInternalServerError) |
| + } |
| + return |
| + } |
| + defer f.Close() |
| + // Stat the OPEN file: its identity is what a remembered tag is checked |
| + // against, and it is the file whose bytes are about to be sent. |
| + if ofi, err := f.Stat(); err == nil && !ofi.IsDir() { |
| + fi = ofi |
| + if tag, ctype, err := staticTags.tag(r.Context(), fp, f, fi); err == nil { |
| + // The type the tag was computed over is the type sent: set here, |
| + // ServeContent leaves it alone. |
| + w.Header().Set("ETag", tag) |
| + w.Header().Set("Content-Type", ctype) |
| + } else { |
| + if !errors.Is(err, errTagBusy) && r.Context().Err() == nil { |
| + // No tag is the honest answer: the response then carries no |
| + // validator and no-cache makes the browser refetch it whole. |
| + log.Printf("edge: static %s/%s: tag %s: %v", s.app, s.channel, p, err) |
| + } |
| + // A failed hash rewinds, but if even that failed the file's |
| + // position is unknown and ServeContent would sniff its type |
| + // from the middle of it. |
| + if _, serr := f.Seek(0, io.SeekStart); serr != nil { |
| + log.Printf("edge: static %s/%s: rewind %s: %v", s.app, s.channel, p, serr) |
| + http.Error(w, "500 Internal Server Error", http.StatusInternalServerError) |
| + return |
| + } |
| + } |
| + } |
| + // ServeContent brings Content-Type, ranges and the conditional-request |
| + // handling; the zero time is "no Last-Modified, and no date to test". |
| + http.ServeContent(w, r, fi.Name(), time.Time{}, f) |
| +} |
| + |
| +// tagCache computes and remembers the entity tag of each extracted file: a |
| +// hash of its bytes and of the one thing besides them that decides what the |
| +// client receives, its Content-Type. |
| +// |
| +// The content, and nothing that merely names it. A version name is reused — |
| +// a pruned version re-shipped under its old name is a new release |
| +// (release.putImmutable), and a hostname can move between two apps that both |
| +// have a "v1" — so a tag made of the name would hand a browser a 304 over |
| +// bytes it does not hold. Hashing also means identical bytes get one tag on |
| +// every edge, and a file a deploy did not touch keeps its tag across it: the |
| +// release that changed one page does not make every browser refetch every |
| +// asset. |
| +// |
| +// The type is hashed in because the same bytes can be two representations. |
| +// /about served from an extensionless file is sniffed as text/plain; the same |
| +// bytes at about/index.html are text/html; and two boxes whose MIME tables |
| +// differ type one .md file differently. A 304 carries no Content-Type, so |
| +// bytes alone would leave the browser rendering the old one. The type is |
| +// resolved here exactly as net/http resolves it — the extension's mapping, |
| +// else a sniff of the first 512 bytes — and handed back with the tag, so the |
| +// type the tag covers is the type the response carries. |
| +// |
| +// WEAK, although such a hash is as strong as a validator gets. The edge gzips |
| +// compressible responses above this handler and weakens a strong tag when it |
| +// does (compress.go, weakenETag) — but only on the 200. A 304 and a HEAD pass |
| +// through as they are, so a strong tag would come back weak on the page and |
| +// strong on its own revalidation, and a cache may not update the one from the |
| +// other. Weak from the start, it reads the same on all three. The cost is |
| +// that If-Range falls back to the whole file. |
| +// |
| +// One cache for the process rather than one per handler: the edge rebuilds |
| +// every static handler on each route-table reload, and a per-handler memo |
| +// would re-hash every site each time a route anywhere on the box changed. |
| +// What makes a shared, long-lived memo safe: |
| +// |
| +// - An entry is checked against the file it was computed FROM, not the path |
| +// it was computed at (os.SameFile, size, mtime). A tree replaced on disk |
| +// under a live handler — cache maintenance, then a re-shipped version of |
| +// the same name extracted again — is new files, and hashes again. |
| +// - A cold file is hashed once however many requests want it: the rest |
| +// wait for the one in flight. |
| +// - A file over tagInline hashes under a semaphore, tagHashers at a time, |
| +// so a sweep of a site's large assets cannot take every core and the disk |
| +// at once. Small files never queue behind them: their work is bounded by |
| +// their size and they hash inline. |
| +// - Nobody waits long. A tag is an optimization, and a request that has |
| +// waited `wait` for a hasher or for someone else's hash is served without |
| +// one (errTagBusy): correct, uncached, and tried again on the next |
| +// request. Only the request doing a hash sees it through, because every |
| +// later request needs the result. |
| +// - It is bounded: past tagCacheMax entries it is emptied and refills on |
| +// demand, which costs hashing and never correctness. |
| +type tagCache struct { |
| + mu sync.Mutex |
| + entries map[string]tagEntry |
| + flights map[string]chan struct{} // closed when the hash for that path is done |
| + sem chan struct{} |
| + wait time.Duration |
| + |
| + // Evidence and a seam for the tests: files hashed, callers currently |
| + // waiting on another's hash, and a hook run inside a hash (nil in |
| + // production) so a test can hold one open. |
| + hashed atomic.Int64 |
| + waiting atomic.Int64 |
| + hold func() |
| +} |
| + |
| +type tagEntry struct { |
| + fi os.FileInfo |
| + tag, ctype string |
| +} |
| + |
| +const ( |
| + tagHashers = 4 |
| + tagInline = 1 << 20 // files up to this size hash without a hasher slot |
| + tagWait = 250 * time.Millisecond |
| + tagCacheMax = 1 << 16 |
| +) |
| + |
| +// errTagBusy: the tag could not be had within the wait budget. The caller |
| +// serves the file untagged. |
| +var errTagBusy = errors.New("static tag: hashers busy") |
| + |
| +func newTagCache() *tagCache { |
| + return &tagCache{ |
| + entries: map[string]tagEntry{}, |
| + flights: map[string]chan struct{}{}, |
| + sem: make(chan struct{}, tagHashers), |
| + wait: tagWait, |
| + } |
| +} |
| + |
| +var staticTags = newTagCache() |
| + |
| +// current reports whether e still describes the file fi is. |
| +func (e tagEntry) current(fi os.FileInfo) bool { |
| + return e.fi != nil && os.SameFile(e.fi, fi) && e.fi.Size() == fi.Size() && e.fi.ModTime().Equal(fi.ModTime()) |
| +} |
| + |
| +// tag returns the entity tag and Content-Type for the open file f at fp, |
| +// hashing it if no current entry exists. f is left positioned at its start. |
| +// ctx and the wait budget bound the WAITING — on another request's hash, or |
| +// on a free hasher — never a hash in progress. |
| +func (c *tagCache) tag(ctx context.Context, fp string, f io.ReadSeeker, fi os.FileInfo) (tag, ctype string, err error) { |
| + var budget <-chan time.Time |
| + for { |
| + c.mu.Lock() |
| + if e, ok := c.entries[fp]; ok && e.current(fi) { |
| + c.mu.Unlock() |
| + return e.tag, e.ctype, nil |
| + } |
| + if budget == nil { |
| + t := time.NewTimer(c.wait) |
| + defer t.Stop() |
| + budget = t.C |
| + } |
| + if flight, ok := c.flights[fp]; ok { |
| + c.mu.Unlock() |
| + c.waiting.Add(1) |
| + select { |
| + case <-flight: |
| + c.waiting.Add(-1) |
| + continue // look again: usually a hit, a miss if the file differs |
| + case <-ctx.Done(): |
| + c.waiting.Add(-1) |
| + return "", "", ctx.Err() |
| + case <-budget: |
| + c.waiting.Add(-1) |
| + return "", "", errTagBusy |
| + } |
| + } |
| + done := make(chan struct{}) |
| + c.flights[fp] = done |
| + c.mu.Unlock() |
| + |
| + tag, ctype, err = c.hash(ctx, budget, f, fi) |
| + c.mu.Lock() |
| + if err == nil { |
| + if len(c.entries) >= tagCacheMax { |
| + c.entries = map[string]tagEntry{} |
| + } |
| + c.entries[fp] = tagEntry{fi: fi, tag: tag, ctype: ctype} |
| + } |
| + delete(c.flights, fp) |
| + c.mu.Unlock() |
| + close(done) |
| + return tag, ctype, err |
| + } |
| +} |
| + |
| +func (c *tagCache) hash(ctx context.Context, budget <-chan time.Time, f io.ReadSeeker, fi os.FileInfo) (tag, ctype string, err error) { |
| + if fi.Size() > tagInline { |
| + select { |
| + case c.sem <- struct{}{}: |
| + defer func() { <-c.sem }() |
| + case <-ctx.Done(): |
| + return "", "", ctx.Err() |
| + case <-budget: |
| + return "", "", errTagBusy |
| + } |
| + } |
| + c.hashed.Add(1) |
| + if c.hold != nil { |
| + c.hold() |
| + } |
| + // From here the file is being read, and it goes back to its start on |
| + // EVERY path out — a failed hash included. The caller serves the file |
| + // either way, and net/http sniffs a Content-Type from wherever the file |
| + // is positioned. |
| + defer func() { |
| + if _, serr := f.Seek(0, io.SeekStart); serr != nil && err == nil { |
| + tag, ctype, err = "", "", serr |
| + } |
| + }() |
| + // net/http's own rule (serveContent): the extension's type, else a sniff |
| + // of the first 512 bytes. |
| + var head [512]byte |
| + n, err := io.ReadFull(f, head[:]) |
| + if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF { |
| + return "", "", err |
| + } |
| + ctype = mime.TypeByExtension(filepath.Ext(fi.Name())) |
| + if ctype == "" { |
| + ctype = http.DetectContentType(head[:n]) |
| + } |
| + h := sha256.New() |
| + io.WriteString(h, ctype) |
| + h.Write([]byte{0}) |
| + h.Write(head[:n]) |
| + if _, err := io.Copy(h, f); err != nil { |
| + return "", "", err |
| + } |
| + return `W/"` + hex.EncodeToString(h.Sum(nil)[:16]) + `"`, ctype, nil |
| +} |
| + |
| +// containsDotDot reports whether a URL path has a ".." element — net/http's |
| +// own test, which http.ServeFile applied before this handler served files |
| +// itself. |
| +func containsDotDot(p string) bool { |
| + if !strings.Contains(p, "..") { |
| + return false |
| + } |
| + for _, el := range strings.FieldsFunc(p, func(r rune) bool { return r == '/' || r == '\\' }) { |
| + if el == ".." { |
| + return true |
| + } |
| + } |
| + return false |
| } |
diff --git a/internal/edge/static_cache_test.go b/internal/edge/static_cache_test.go| new file mode 100644 |
| index 000000000..ff7d9178f |
| --- /dev/null |
| +++ b/internal/edge/static_cache_test.go |
| @@ -0,0 +1,596 @@ |
| +package edge |
| + |
| +import ( |
| + "context" |
| + "errors" |
| + "fmt" |
| + "io" |
| + "mime" |
| + "net/http" |
| + "net/http/httptest" |
| + "os" |
| + "path/filepath" |
| + "strings" |
| + "sync" |
| + "testing" |
| + "time" |
| + |
| + "github.com/carlosframework/platform/internal/registry" |
| + "github.com/carlosframework/platform/internal/release" |
| +) |
| + |
| +// reqStatic is one request the way a browser holding a cached copy makes it: |
| +// no redirect following, no transparent gzip (an explicit Accept-Encoding |
| +// turns the client's own off), and the validators in hdr. |
| +func reqStatic(t *testing.T, method, base, path string, hdr map[string]string) (*http.Response, string) { |
| + t.Helper() |
| + req, err := http.NewRequest(method, base+path, nil) |
| + if err != nil { |
| + t.Fatalf("new request: %v", err) |
| + } |
| + req.Host = "site.example.com" |
| + for k, v := range hdr { |
| + req.Header.Set(k, v) |
| + } |
| + client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} |
| + resp, err := client.Do(req) |
| + if err != nil { |
| + t.Fatalf("%s %s%s: %v", method, base, path, err) |
| + } |
| + defer resp.Body.Close() |
| + b, err := io.ReadAll(resp.Body) |
| + if err != nil { |
| + t.Fatalf("%s %s%s: read body: %v", method, base, path, err) |
| + } |
| + return resp, string(b) |
| +} |
| + |
| +// siteServer ships files as version of app into its own store and serves it |
| +// from a fresh handler, behind the same compression wrapper the edge applies. |
| +func siteServer(t *testing.T, app, version string, files map[string]string) *httptest.Server { |
| + t.Helper() |
| + srv, _ := siteServerTree(t, app, version, files) |
| + return srv |
| +} |
| + |
| +// siteServerTree is siteServer that also says where version's tree is |
| +// extracted, for the tests that reach under the handler. |
| +func siteServerTree(t *testing.T, app, version string, files map[string]string) (*httptest.Server, string) { |
| + t.Helper() |
| + store := release.NewDirStore(filepath.Join(t.TempDir(), "store")) |
| + priv, pub := testKeys(t) |
| + shipSite(t, store, app, version, files) |
| + if err := release.Promote(context.Background(), store, priv, release.OpsAccount, app, "canary/x", version); err != nil { |
| + t.Fatalf("Promote %s: %v", version, err) |
| + } |
| + rt := registry.Route{Host: "site.example.com", Kind: "static", App: app, Channel: "canary/x", Status: "ready"} |
| + h := newStaticHandler(rt, store, pub, filepath.Join(t.TempDir(), "cache")) |
| + h.(*staticSite).recheck = 0 |
| + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| + if gw := newGzipWriter(w, r); gw != nil { |
| + w = gw |
| + defer gw.finish() |
| + } |
| + h.ServeHTTP(w, r) |
| + })) |
| + t.Cleanup(srv.Close) |
| + return srv, h.(*staticSite).dir(version) |
| +} |
| + |
| +// A static response used to carry Last-Modified and nothing else, and a |
| +// browser handed only that guesses a freshness lifetime — typically a tenth of |
| +// the file's age — and reuses its copy without asking. A site last deployed a |
| +// month ago kept showing the old page for days after the next deploy |
| +// (discussion #195). no-cache makes every load ask; the ETag makes the answer |
| +// a cheap 304 until the bytes actually change. |
| +func TestStaticResponsesMakeTheBrowserRevalidate(t *testing.T) { |
| + _, _, _, srv := staticFixture(t) |
| + |
| + for _, path := range []string{"/", "/docs/page.html", "/style.css"} { |
| + resp, _ := reqStatic(t, "GET", srv.URL, path, nil) |
| + if resp.StatusCode != http.StatusOK { |
| + t.Fatalf("GET %s = %d, want 200", path, resp.StatusCode) |
| + } |
| + if got := resp.Header.Get("Cache-Control"); got != "no-cache" { |
| + t.Errorf("GET %s Cache-Control = %q, want %q", path, got, "no-cache") |
| + } |
| + if got := resp.Header.Get("ETag"); !strings.HasPrefix(got, `W/"`) || len(got) < 20 { |
| + t.Errorf("GET %s ETag = %q, want a weak content tag", path, got) |
| + } |
| + // The mtime of an extracted file is when THIS box unpacked it. It is |
| + // not a fact about the content and must not be offered as one. |
| + if got := resp.Header.Get("Last-Modified"); got != "" { |
| + t.Errorf("GET %s Last-Modified = %q, want none", path, got) |
| + } |
| + } |
| + |
| + // The tree's own 404 must not be cached on a guess either: the page it |
| + // stands in for may exist in the next deploy. It names no representation, |
| + // so it carries no validator. |
| + resp, _ := reqStatic(t, "GET", srv.URL, "/missing.html", nil) |
| + if resp.StatusCode != http.StatusNotFound { |
| + t.Fatalf("GET /missing.html = %d, want 404", resp.StatusCode) |
| + } |
| + if got := resp.Header.Get("Cache-Control"); got != "no-cache" { |
| + t.Errorf("404 Cache-Control = %q, want %q", got, "no-cache") |
| + } |
| + if got := resp.Header.Get("ETag"); got != "" { |
| + t.Errorf("404 ETag = %q, want none", got) |
| + } |
| +} |
| + |
| +// The validator is the file's CONTENT. Unchanged bytes revalidate to a 304 — |
| +// across a deploy too, so a release that touched one page does not make every |
| +// browser refetch every asset — and changed bytes answer 200, in both |
| +// directions: a rollback is as much a change as a deploy. |
| +func TestStaticValidatorFollowsTheContent(t *testing.T) { |
| + store, _, priv, srv := staticFixture(t) |
| + promote := func(ver string) { |
| + t.Helper() |
| + if err := release.Promote(context.Background(), store, priv, release.OpsAccount, "site", "canary/x", ver); err != nil { |
| + t.Fatalf("Promote %s: %v", ver, err) |
| + } |
| + } |
| + tagOf := func(path string) string { |
| + t.Helper() |
| + resp, _ := reqStatic(t, "GET", srv.URL, path, nil) |
| + tag := resp.Header.Get("ETag") |
| + if resp.StatusCode != http.StatusOK || tag == "" { |
| + t.Fatalf("GET %s = %d with ETag %q", path, resp.StatusCode, tag) |
| + } |
| + return tag |
| + } |
| + homeV1, cssV1 := tagOf("/"), tagOf("/style.css") |
| + if homeV1 == cssV1 { |
| + t.Fatalf("two different files share the tag %q", homeV1) |
| + } |
| + |
| + resp, _ := reqStatic(t, "GET", srv.URL, "/", map[string]string{"If-None-Match": homeV1}) |
| + if resp.StatusCode != http.StatusNotModified { |
| + t.Fatalf("revalidating unchanged content = %d, want 304", resp.StatusCode) |
| + } |
| + // A 304 carries what its 200 would have: the cache updates from it. |
| + if got := resp.Header.Get("ETag"); got != homeV1 { |
| + t.Errorf("304 ETag = %q, want the 200's %q", got, homeV1) |
| + } |
| + if got := resp.Header.Get("Cache-Control"); got != "no-cache" { |
| + t.Errorf("304 Cache-Control = %q, want %q", got, "no-cache") |
| + } |
| + |
| + // v2 changes the home page and leaves the stylesheet byte-identical. |
| + shipSite(t, store, "site", "v2", map[string]string{"index.html": "home v2", "style.css": "body{}"}) |
| + promote("v2") |
| + resp, body := reqStatic(t, "GET", srv.URL, "/", map[string]string{"If-None-Match": homeV1}) |
| + if resp.StatusCode != http.StatusOK || body != "home v2" { |
| + t.Fatalf("revalidating a changed page after a deploy = %d %q, want 200 %q", resp.StatusCode, body, "home v2") |
| + } |
| + homeV2 := resp.Header.Get("ETag") |
| + if homeV2 == homeV1 { |
| + t.Fatalf("the changed page kept the tag %q", homeV1) |
| + } |
| + resp, _ = reqStatic(t, "GET", srv.URL, "/style.css", map[string]string{"If-None-Match": cssV1}) |
| + if resp.StatusCode != http.StatusNotModified { |
| + t.Fatalf("revalidating an UNCHANGED file after a deploy = %d, want 304", resp.StatusCode) |
| + } |
| + |
| + promote("v1") // roll back |
| + resp, body = reqStatic(t, "GET", srv.URL, "/", map[string]string{"If-None-Match": homeV2}) |
| + if resp.StatusCode != http.StatusOK || body != "home v1" { |
| + t.Fatalf("revalidating after a rollback = %d %q, want 200 %q", resp.StatusCode, body, "home v1") |
| + } |
| + if got := resp.Header.Get("ETag"); got != homeV1 { |
| + t.Errorf("the rolled-back page's tag = %q, want its original %q", got, homeV1) |
| + } |
| +} |
| + |
| +// A browser whose copy predates the ETag holds only the old Last-Modified and |
| +// revalidates on the date alone. Against an extraction mtime that answer is |
| +// wrong after a rollback — the tree rolled back to was unpacked BEFORE the one |
| +// the browser holds, so "modified since?" says no. With no date offered there |
| +// is nothing to compare, and the browser gets the bytes. |
| +func TestStaticDateOnlyRevalidationGetsTheBytes(t *testing.T) { |
| + _, _, _, srv := staticFixture(t) |
| + resp, body := reqStatic(t, "GET", srv.URL, "/", map[string]string{ |
| + "If-Modified-Since": "Fri, 01 Jan 2100 00:00:00 GMT", |
| + }) |
| + if resp.StatusCode != http.StatusOK || body != "home v1" { |
| + t.Fatalf("date-only revalidation = %d %q, want 200 %q", resp.StatusCode, body, "home v1") |
| + } |
| +} |
| + |
| +// A version NAME is not a fact about the bytes. A pruned version can be |
| +// re-shipped under its old name with different content, and a hostname can |
| +// move between two apps that both have a "v1" — in either case a tag made of |
| +// the name would let a browser keep the old page on a 304. |
| +func TestStaticSameVersionNameDifferentContent(t *testing.T) { |
| + a := siteServer(t, "site", "v1", map[string]string{"index.html": "the first site"}) |
| + b := siteServer(t, "other", "v1", map[string]string{"index.html": "a different site"}) |
| + |
| + respA, _ := reqStatic(t, "GET", a.URL, "/", nil) |
| + tagA := respA.Header.Get("ETag") |
| + if tagA == "" { |
| + t.Fatal("no ETag from the first site") |
| + } |
| + resp, body := reqStatic(t, "GET", b.URL, "/", map[string]string{"If-None-Match": tagA}) |
| + if resp.StatusCode != http.StatusOK || body != "a different site" { |
| + t.Fatalf("revalidating one v1 against another app's v1 = %d %q, want 200 %q", resp.StatusCode, body, "a different site") |
| + } |
| + |
| + // And identical bytes agree wherever they are served from — two edges |
| + // unpacking the same release must hand out the same tag. |
| + c := siteServer(t, "third", "v9", map[string]string{"index.html": "the first site"}) |
| + resp, _ = reqStatic(t, "GET", c.URL, "/", map[string]string{"If-None-Match": tagA}) |
| + if resp.StatusCode != http.StatusNotModified { |
| + t.Fatalf("identical bytes from another handler = %d, want 304", resp.StatusCode) |
| + } |
| +} |
| + |
| +// The edge gzips a compressible response and, because gzipped bytes are a |
| +// different representation, weakens a strong ETag as it does — but only on |
| +// the 200. A 304 and a HEAD pass through untouched, so a strong tag would |
| +// come back weak on the page and strong on its revalidation, and a cache may |
| +// not update the one from the other. A tag that is weak from the start reads |
| +// the same on all three. |
| +func TestStaticTagIsTheSameCompressedRevalidatedAndOnHead(t *testing.T) { |
| + page := "<html>" + strings.Repeat("a page big enough to be worth compressing. ", 200) + "</html>" |
| + srv := siteServer(t, "site", "v1", map[string]string{"index.html": page}) |
| + gz := map[string]string{"Accept-Encoding": "gzip"} |
| + |
| + resp, _ := reqStatic(t, "GET", srv.URL, "/", gz) |
| + if resp.StatusCode != http.StatusOK || resp.Header.Get("Content-Encoding") != "gzip" { |
| + t.Fatalf("GET / = %d, Content-Encoding %q; want a gzipped 200", resp.StatusCode, resp.Header.Get("Content-Encoding")) |
| + } |
| + tag := resp.Header.Get("ETag") |
| + if !strings.HasPrefix(tag, `W/"`) { |
| + t.Fatalf("gzipped 200 ETag = %q, want a weak tag", tag) |
| + } |
| + |
| + resp, _ = reqStatic(t, "GET", srv.URL, "/", map[string]string{"Accept-Encoding": "gzip", "If-None-Match": tag}) |
| + if resp.StatusCode != http.StatusNotModified { |
| + t.Fatalf("revalidating the gzipped page = %d, want 304", resp.StatusCode) |
| + } |
| + if got := resp.Header.Get("ETag"); got != tag { |
| + t.Errorf("304 ETag = %q, want the 200's %q", got, tag) |
| + } |
| + |
| + resp, _ = reqStatic(t, "HEAD", srv.URL, "/", gz) |
| + if got := resp.Header.Get("ETag"); got != tag { |
| + t.Errorf("HEAD ETag = %q, want the GET's %q", got, tag) |
| + } |
| + // Asked for by name: with no Accept-Encoding of its own the Go client |
| + // requests gzip and hides that it did. |
| + resp, body := reqStatic(t, "GET", srv.URL, "/", map[string]string{"Accept-Encoding": "identity"}) |
| + if resp.StatusCode != http.StatusOK || body != page || resp.Header.Get("Content-Encoding") != "" { |
| + t.Fatalf("identity GET = %d, Content-Encoding %q, %d bytes; want the plain page", resp.StatusCode, resp.Header.Get("Content-Encoding"), len(body)) |
| + } |
| + if got := resp.Header.Get("ETag"); got != tag { |
| + t.Errorf("identity GET ETag = %q, want the gzipped GET's %q", got, tag) |
| + } |
| +} |
| + |
| +// What http.ServeFile gave for free and this handler must keep now that it |
| +// serves the file itself: the canonical redirect off /index.html, ranges, and |
| +// the refusal of a path with a ".." element. |
| +func TestStaticKeepsServeFileBehaviour(t *testing.T) { |
| + _, _, _, srv := staticFixture(t) |
| + |
| + resp, _ := reqStatic(t, "GET", srv.URL, "/index.html?x=1", nil) |
| + if resp.StatusCode != http.StatusMovedPermanently || resp.Header.Get("Location") != "./?x=1" { |
| + t.Fatalf("GET /index.html = %d, Location %q; want 301 to ./?x=1", resp.StatusCode, resp.Header.Get("Location")) |
| + } |
| + |
| + resp, body := reqStatic(t, "GET", srv.URL, "/", map[string]string{"Range": "bytes=0-3"}) |
| + if resp.StatusCode != http.StatusPartialContent || body != "home" { |
| + t.Fatalf("ranged GET = %d %q, want 206 %q", resp.StatusCode, body, "home") |
| + } |
| + // A weak tag cannot vouch for a byte range: If-Range falls back to the |
| + // whole file, which is the safe answer and the documented cost. |
| + tag := resp.Header.Get("ETag") |
| + resp, body = reqStatic(t, "GET", srv.URL, "/", map[string]string{"Range": "bytes=0-3", "If-Range": tag}) |
| + if resp.StatusCode != http.StatusOK || body != "home v1" { |
| + t.Fatalf("If-Range with a weak tag = %d %q, want the full 200", resp.StatusCode, body) |
| + } |
| + |
| + req, err := http.NewRequest("GET", srv.URL+"/", nil) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + req.Host = "site.example.com" |
| + req.URL.Opaque = "/docs/../index.html" // sent as written, not cleaned by the client |
| + resp2, err := http.DefaultClient.Do(req) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + resp2.Body.Close() |
| + if resp2.StatusCode != http.StatusBadRequest { |
| + t.Fatalf("a path with a .. element = %d, want 400", resp2.StatusCode) |
| + } |
| +} |
| + |
| +// The same bytes can be two representations. /about as an extensionless file |
| +// is sniffed as text/plain; redeployed as about/index.html it is text/html, |
| +// byte for byte the same. A tag made of the bytes alone would answer 304 and |
| +// leave the browser rendering markup as text. |
| +func TestStaticTagCoversTheContentType(t *testing.T) { |
| + store, _, priv, srv := staticFixture(t) |
| + const same = "Hello <em>world</em>" |
| + shipSite(t, store, "site", "v2", map[string]string{"index.html": "home", "about": same}) |
| + if err := release.Promote(context.Background(), store, priv, release.OpsAccount, "site", "canary/x", "v2"); err != nil { |
| + t.Fatalf("Promote v2: %v", err) |
| + } |
| + resp, _ := reqStatic(t, "GET", srv.URL, "/about", nil) |
| + plainTag, plainType := resp.Header.Get("ETag"), resp.Header.Get("Content-Type") |
| + if resp.StatusCode != http.StatusOK || !strings.HasPrefix(plainType, "text/plain") { |
| + t.Fatalf("GET /about = %d %q, want a text/plain 200", resp.StatusCode, plainType) |
| + } |
| + |
| + shipSite(t, store, "site", "v3", map[string]string{"index.html": "home", "about/index.html": same}) |
| + if err := release.Promote(context.Background(), store, priv, release.OpsAccount, "site", "canary/x", "v3"); err != nil { |
| + t.Fatalf("Promote v3: %v", err) |
| + } |
| + resp, body := reqStatic(t, "GET", srv.URL, "/about", map[string]string{"If-None-Match": plainTag}) |
| + if resp.StatusCode != http.StatusOK || body != same || !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") { |
| + t.Fatalf("revalidating /about after it became HTML = %d %q; want a text/html 200", resp.StatusCode, resp.Header.Get("Content-Type")) |
| + } |
| +} |
| + |
| +// openTagged writes body at dir/name and opens it the way serveTree does. |
| +func openTagged(t *testing.T, dir, name, body string) (string, *os.File, os.FileInfo) { |
| + t.Helper() |
| + fp := filepath.Join(dir, name) |
| + if err := os.WriteFile(fp, []byte(body), 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| + f, err := os.Open(fp) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { f.Close() }) |
| + fi, err := f.Stat() |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + return fp, f, fi |
| +} |
| + |
| +// A cold file is hashed ONCE however many requests arrive for it together. |
| +// Without that, a burst of requests against one large uncached asset — HEADs |
| +// and one-byte ranges included — would each read and hash the whole file |
| +// before sending a byte. |
| +// |
| +// The overlap is forced, not hoped for: the first hash is held until every |
| +// other caller is known to be waiting on it. |
| +func TestTagCacheHashesAColdFileOnceForOverlappingRequests(t *testing.T) { |
| + c := newTagCache() |
| + c.wait = time.Minute |
| + held, release := make(chan struct{}), make(chan struct{}) |
| + var heldOnce, releaseOnce sync.Once |
| + c.hold = func() { heldOnce.Do(func() { close(held) }); <-release } |
| + letGo := func() { releaseOnce.Do(func() { close(release) }) } |
| + defer letGo() // a failure below must not strand the callers |
| + dir := t.TempDir() |
| + if err := os.WriteFile(filepath.Join(dir, "big.bin"), []byte(strings.Repeat("x", 4096)), 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| + const callers = 12 |
| + tags := make([]string, callers) |
| + errs := make([]error, callers) |
| + var wg sync.WaitGroup |
| + call := func(i int) { |
| + defer wg.Done() |
| + fp := filepath.Join(dir, "big.bin") |
| + f, err := os.Open(fp) |
| + if err != nil { |
| + errs[i] = err |
| + return |
| + } |
| + defer f.Close() |
| + fi, err := f.Stat() |
| + if err != nil { |
| + errs[i] = err |
| + return |
| + } |
| + tags[i], _, errs[i] = c.tag(context.Background(), fp, f, fi) |
| + } |
| + wg.Add(1) |
| + go call(0) |
| + <-held // the first caller is inside its hash |
| + for i := 1; i < callers; i++ { |
| + wg.Add(1) |
| + go call(i) |
| + } |
| + deadline := time.Now().Add(10 * time.Second) |
| + for c.waiting.Load() != callers-1 { |
| + if time.Now().After(deadline) { |
| + t.Fatalf("%d of %d callers are waiting on the hash in flight", c.waiting.Load(), callers-1) |
| + } |
| + time.Sleep(time.Millisecond) |
| + } |
| + letGo() |
| + wg.Wait() |
| + for i := range tags { |
| + if errs[i] != nil || tags[i] == "" || tags[i] != tags[0] { |
| + t.Fatalf("caller %d = %q, %v; want every caller to get %q", i, tags[i], errs[i], tags[0]) |
| + } |
| + } |
| + if got := c.hashed.Load(); got != 1 { |
| + t.Fatalf("the file was hashed %d times for %d overlapping callers, want once", got, callers) |
| + } |
| +} |
| + |
| +// Large files hash a few at a time, and a request that cannot get a turn — |
| +// or whose file is still being hashed for someone else — is not made to wait |
| +// for it: past a short budget it is served without a tag. A small file never |
| +// queues behind them at all. Otherwise four requests for four large cold |
| +// assets would hold up the first load of every other static site on the box. |
| +func TestTagCacheDoesNotQueueBehindLargeHashes(t *testing.T) { |
| + c := newTagCache() |
| + c.wait = 20 * time.Millisecond |
| + dir := t.TempDir() |
| + for i := 0; i < tagHashers; i++ { |
| + c.sem <- struct{}{} // every hasher busy with someone's large asset |
| + } |
| + |
| + fp, f, fi := openTagged(t, dir, "big.bin", strings.Repeat("x", tagInline+1)) |
| + if _, _, err := c.tag(context.Background(), fp, f, fi); !errors.Is(err, errTagBusy) { |
| + t.Fatalf("a large file with no free hasher = %v, want errTagBusy", err) |
| + } |
| + if n := len(c.flights); n != 0 { |
| + t.Fatalf("%d flights left registered after giving up", n) |
| + } |
| + |
| + fp, f, fi = openTagged(t, dir, "page.html", "<p>a small page</p>") |
| + tag, ctype, err := c.tag(context.Background(), fp, f, fi) |
| + if err != nil || tag == "" || !strings.HasPrefix(ctype, "text/html") { |
| + t.Fatalf("a small file while every hasher is busy = %q %q %v; want it tagged at once", tag, ctype, err) |
| + } |
| + |
| + // Once a hasher frees up the large file is tagged, and stays tagged. |
| + // The budget is lifted first: at 20ms a stalled scheduler could find the |
| + // timer and the free hasher ready together, and pick the timer. |
| + <-c.sem |
| + c.wait = time.Minute |
| + fp, f, fi = openTagged(t, dir, "big.bin", strings.Repeat("x", tagInline+1)) |
| + if tag, _, err := c.tag(context.Background(), fp, f, fi); err != nil || tag == "" { |
| + t.Fatalf("a large file with a free hasher = %q, %v", tag, err) |
| + } |
| +} |
| + |
| +// The tag covers the Content-Type the response will carry, not the extension |
| +// it was derived from: two boxes with different MIME tables type the same |
| +// bytes differently, and a 304 from one must not confirm the other's page. |
| +func TestTagCacheTagCoversTheResolvedContentType(t *testing.T) { |
| + c := newTagCache() |
| + const body = "# notes\n\nsome text\n" |
| + // The MIME table is process-wide and an added type stays added, so each |
| + // run takes an extension of its own (-count=N would otherwise find the |
| + // last run's mapping in place of an unknown one). |
| + ext := fmt.Sprintf(".zzcarlos%d", time.Now().UnixNano()) |
| + fp, f, fi := openTagged(t, t.TempDir(), "notes"+ext, body) |
| + sniffedTag, sniffedType, err := c.tag(context.Background(), fp, f, fi) |
| + if err != nil || !strings.HasPrefix(sniffedType, "text/plain") { |
| + t.Fatalf("an unknown extension = %q %v, want it sniffed as text/plain", sniffedType, err) |
| + } |
| + |
| + // "Another box": the same bytes under the same name, with the type known. |
| + if err := mime.AddExtensionType(ext, "text/markdown; charset=utf-8"); err != nil { |
| + t.Fatal(err) |
| + } |
| + fp, f, fi = openTagged(t, t.TempDir(), "notes"+ext, body) |
| + knownTag, knownType, err := c.tag(context.Background(), fp, f, fi) |
| + if err != nil || !strings.HasPrefix(knownType, "text/markdown") { |
| + t.Fatalf("a mapped extension = %q %v, want text/markdown", knownType, err) |
| + } |
| + if knownTag == sniffedTag { |
| + t.Fatalf("the same bytes typed %q and %q share the tag %s", sniffedType, knownType, knownTag) |
| + } |
| +} |
| + |
| +// A remembered tag describes the file it was computed from, not the path. If |
| +// the tree is replaced on disk under a live handler — cache maintenance, then |
| +// another handler unpacking a re-shipped version of the same name — the old |
| +// tag must not be handed out over the new bytes, nor a 304 against it. |
| +func TestStaticTagDoesNotOutliveTheFileItDescribes(t *testing.T) { |
| + srv, tree := siteServerTree(t, "site", "v1", map[string]string{"index.html": "home", "style.css": "body{}"}) |
| + resp, _ := reqStatic(t, "GET", srv.URL, "/style.css", nil) |
| + oldTag := resp.Header.Get("ETag") |
| + if oldTag == "" { |
| + t.Fatal("no ETag for /style.css") |
| + } |
| + |
| + // Replace the file the way a fresh extraction would — a new file renamed |
| + // into place — and make it the hardest case to notice: the same length |
| + // and the same mtime, so only the file's identity gives it away. |
| + fp := filepath.Join(tree, "style.css") |
| + before, err := os.Stat(fp) |
| + if err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.WriteFile(fp+".new", []byte("main{}"), 0o644); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.Chtimes(fp+".new", before.ModTime(), before.ModTime()); err != nil { |
| + t.Fatal(err) |
| + } |
| + if err := os.Rename(fp+".new", fp); err != nil { |
| + t.Fatal(err) |
| + } |
| + |
| + resp, body := reqStatic(t, "GET", srv.URL, "/style.css", map[string]string{"If-None-Match": oldTag}) |
| + if resp.StatusCode != http.StatusOK || body != "main{}" { |
| + t.Fatalf("revalidating against a replaced file = %d %q, want 200 with the new bytes", resp.StatusCode, body) |
| + } |
| + if got := resp.Header.Get("ETag"); got == "" || got == oldTag { |
| + t.Fatalf("ETag over the replaced file = %q, want a new tag (old %q)", got, oldTag) |
| + } |
| +} |
| + |
| +// A file that is there and cannot be read is not a missing page. ServeFile |
| +// said 403, and an operator reading the access log must still be able to tell |
| +// a broken cache from a mistyped URL. |
| +func TestStaticUnreadableFileIsNotA404(t *testing.T) { |
| + if os.Geteuid() == 0 { |
| + t.Skip("root reads through file modes") |
| + } |
| + srv, tree := siteServerTree(t, "site", "v1", map[string]string{"index.html": "home", "style.css": "body{}"}) |
| + reqStatic(t, "GET", srv.URL, "/", nil) // extract the tree |
| + fp := filepath.Join(tree, "style.css") |
| + if err := os.Chmod(fp, 0); err != nil { |
| + t.Fatal(err) |
| + } |
| + t.Cleanup(func() { os.Chmod(fp, 0o644) }) |
| + resp, _ := reqStatic(t, "GET", srv.URL, "/style.css", nil) |
| + if resp.StatusCode != http.StatusForbidden { |
| + t.Fatalf("GET of an unreadable file = %d, want 403", resp.StatusCode) |
| + } |
| +} |
| + |
| +// failingFile reads like a file whose later blocks are bad: the first good |
| +// bytes come back, then every read fails. Seeks work. |
| +type failingFile struct { |
| + data []byte |
| + good int |
| + pos int64 |
| +} |
| + |
| +func (f *failingFile) Read(p []byte) (int, error) { |
| + if f.pos >= int64(f.good) { |
| + return 0, errors.New("input/output error") |
| + } |
| + n := copy(p, f.data[f.pos:f.good]) |
| + f.pos += int64(n) |
| + return n, nil |
| +} |
| + |
| +func (f *failingFile) Seek(off int64, whence int) (int64, error) { |
| + switch whence { |
| + case io.SeekStart: |
| + f.pos = off |
| + case io.SeekCurrent: |
| + f.pos += off |
| + case io.SeekEnd: |
| + f.pos = int64(len(f.data)) + off |
| + } |
| + return f.pos, nil |
| +} |
| + |
| +// A hash that fails part-way must hand the file back at its start. The |
| +// handler then serves it untagged, and net/http sniffs the Content-Type from |
| +// wherever the file is positioned: left mid-file, an extensionless image |
| +// would go out as text/plain. |
| +func TestTagCacheRewindsTheFileWhenHashingFails(t *testing.T) { |
| + c := newTagCache() |
| + _, _, fi := openTagged(t, t.TempDir(), "picture", "placeholder") |
| + for _, good := range []int{100, 2000} { // fails inside the sniffed prefix, and after it |
| + f := &failingFile{data: make([]byte, 4096), good: good} |
| + if _, _, err := c.tag(context.Background(), fmt.Sprintf("/nowhere/picture-%d", good), f, fi); err == nil { |
| + t.Fatalf("hashing a file that fails after %d bytes reported no error", good) |
| + } |
| + if f.pos != 0 { |
| + t.Fatalf("after a failed hash (%d good bytes) the file is at offset %d, want 0", good, f.pos) |
| + } |
| + } |
| + if n := len(c.entries); n != 0 { |
| + t.Fatalf("%d tags remembered from hashes that failed", n) |
| + } |
| +} |