| 1 | package edge |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "fmt" |
| 7 | "io" |
| 8 | "mime" |
| 9 | "net/http" |
| 10 | "net/http/httptest" |
| 11 | "os" |
| 12 | "path/filepath" |
| 13 | "strings" |
| 14 | "sync" |
| 15 | "testing" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/registry" |
| 19 | "github.com/carlosframework/platform/internal/release" |
| 20 | ) |
| 21 | |
| 22 | // reqStatic is one request the way a browser holding a cached copy makes it: |
| 23 | // no redirect following, no transparent gzip (an explicit Accept-Encoding |
| 24 | // turns the client's own off), and the validators in hdr. |
| 25 | func reqStatic(t *testing.T, method, base, path string, hdr map[string]string) (*http.Response, string) { |
| 26 | t.Helper() |
| 27 | req, err := http.NewRequest(method, base+path, nil) |
| 28 | if err != nil { |
| 29 | t.Fatalf("new request: %v", err) |
| 30 | } |
| 31 | req.Host = "site.example.com" |
| 32 | for k, v := range hdr { |
| 33 | req.Header.Set(k, v) |
| 34 | } |
| 35 | client := &http.Client{CheckRedirect: func(*http.Request, []*http.Request) error { return http.ErrUseLastResponse }} |
| 36 | resp, err := client.Do(req) |
| 37 | if err != nil { |
| 38 | t.Fatalf("%s %s%s: %v", method, base, path, err) |
| 39 | } |
| 40 | defer resp.Body.Close() |
| 41 | b, err := io.ReadAll(resp.Body) |
| 42 | if err != nil { |
| 43 | t.Fatalf("%s %s%s: read body: %v", method, base, path, err) |
| 44 | } |
| 45 | return resp, string(b) |
| 46 | } |
| 47 | |
| 48 | // siteServer ships files as version of app into its own store and serves it |
| 49 | // from a fresh handler, behind the same compression wrapper the edge applies. |
| 50 | func siteServer(t *testing.T, app, version string, files map[string]string) *httptest.Server { |
| 51 | t.Helper() |
| 52 | srv, _ := siteServerTree(t, app, version, files) |
| 53 | return srv |
| 54 | } |
| 55 | |
| 56 | // siteServerTree is siteServer that also says where version's tree is |
| 57 | // extracted, for the tests that reach under the handler. |
| 58 | func siteServerTree(t *testing.T, app, version string, files map[string]string) (*httptest.Server, string) { |
| 59 | t.Helper() |
| 60 | store := release.NewDirStore(filepath.Join(t.TempDir(), "store")) |
| 61 | priv, pub := testKeys(t) |
| 62 | shipSite(t, store, app, version, files) |
| 63 | if err := release.Promote(context.Background(), store, priv, release.OpsAccount, app, "canary/x", version); err != nil { |
| 64 | t.Fatalf("Promote %s: %v", version, err) |
| 65 | } |
| 66 | rt := registry.Route{Host: "site.example.com", Kind: "static", App: app, Channel: "canary/x", Status: "ready"} |
| 67 | h := newStaticHandler(rt, store, pub, filepath.Join(t.TempDir(), "cache")) |
| 68 | h.(*staticSite).recheck = 0 |
| 69 | srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { |
| 70 | if gw := newGzipWriter(w, r); gw != nil { |
| 71 | w = gw |
| 72 | defer gw.finish() |
| 73 | } |
| 74 | h.ServeHTTP(w, r) |
| 75 | })) |
| 76 | t.Cleanup(srv.Close) |
| 77 | return srv, h.(*staticSite).dir(siteDirOf(t, store, app, version)) |
| 78 | } |
| 79 | |
| 80 | // A static response used to carry Last-Modified and nothing else, and a |
| 81 | // browser handed only that guesses a freshness lifetime — typically a tenth of |
| 82 | // the file's age — and reuses its copy without asking. A site last deployed a |
| 83 | // month ago kept showing the old page for days after the next deploy |
| 84 | // (discussion #195). no-cache makes every load ask; the ETag makes the answer |
| 85 | // a cheap 304 until the bytes actually change. |
| 86 | func TestStaticResponsesMakeTheBrowserRevalidate(t *testing.T) { |
| 87 | _, _, _, srv := staticFixture(t) |
| 88 | |
| 89 | for _, path := range []string{"/", "/docs/page.html", "/style.css"} { |
| 90 | resp, _ := reqStatic(t, "GET", srv.URL, path, nil) |
| 91 | if resp.StatusCode != http.StatusOK { |
| 92 | t.Fatalf("GET %s = %d, want 200", path, resp.StatusCode) |
| 93 | } |
| 94 | if got := resp.Header.Get("Cache-Control"); got != "no-cache" { |
| 95 | t.Errorf("GET %s Cache-Control = %q, want %q", path, got, "no-cache") |
| 96 | } |
| 97 | if got := resp.Header.Get("ETag"); !strings.HasPrefix(got, `W/"`) || len(got) < 20 { |
| 98 | t.Errorf("GET %s ETag = %q, want a weak content tag", path, got) |
| 99 | } |
| 100 | // The mtime of an extracted file is when THIS box unpacked it. It is |
| 101 | // not a fact about the content and must not be offered as one. |
| 102 | if got := resp.Header.Get("Last-Modified"); got != "" { |
| 103 | t.Errorf("GET %s Last-Modified = %q, want none", path, got) |
| 104 | } |
| 105 | } |
| 106 | |
| 107 | // The tree's own 404 must not be cached on a guess either: the page it |
| 108 | // stands in for may exist in the next deploy. It names no representation, |
| 109 | // so it carries no validator. |
| 110 | resp, _ := reqStatic(t, "GET", srv.URL, "/missing.html", nil) |
| 111 | if resp.StatusCode != http.StatusNotFound { |
| 112 | t.Fatalf("GET /missing.html = %d, want 404", resp.StatusCode) |
| 113 | } |
| 114 | if got := resp.Header.Get("Cache-Control"); got != "no-cache" { |
| 115 | t.Errorf("404 Cache-Control = %q, want %q", got, "no-cache") |
| 116 | } |
| 117 | if got := resp.Header.Get("ETag"); got != "" { |
| 118 | t.Errorf("404 ETag = %q, want none", got) |
| 119 | } |
| 120 | } |
| 121 | |
| 122 | // The validator is the file's CONTENT. Unchanged bytes revalidate to a 304 — |
| 123 | // across a deploy too, so a release that touched one page does not make every |
| 124 | // browser refetch every asset — and changed bytes answer 200, in both |
| 125 | // directions: a rollback is as much a change as a deploy. |
| 126 | func TestStaticValidatorFollowsTheContent(t *testing.T) { |
| 127 | store, _, priv, srv := staticFixture(t) |
| 128 | promote := func(ver string) { |
| 129 | t.Helper() |
| 130 | if err := release.Promote(context.Background(), store, priv, release.OpsAccount, "site", "canary/x", ver); err != nil { |
| 131 | t.Fatalf("Promote %s: %v", ver, err) |
| 132 | } |
| 133 | } |
| 134 | tagOf := func(path string) string { |
| 135 | t.Helper() |
| 136 | resp, _ := reqStatic(t, "GET", srv.URL, path, nil) |
| 137 | tag := resp.Header.Get("ETag") |
| 138 | if resp.StatusCode != http.StatusOK || tag == "" { |
| 139 | t.Fatalf("GET %s = %d with ETag %q", path, resp.StatusCode, tag) |
| 140 | } |
| 141 | return tag |
| 142 | } |
| 143 | homeV1, cssV1 := tagOf("/"), tagOf("/style.css") |
| 144 | if homeV1 == cssV1 { |
| 145 | t.Fatalf("two different files share the tag %q", homeV1) |
| 146 | } |
| 147 | |
| 148 | resp, _ := reqStatic(t, "GET", srv.URL, "/", map[string]string{"If-None-Match": homeV1}) |
| 149 | if resp.StatusCode != http.StatusNotModified { |
| 150 | t.Fatalf("revalidating unchanged content = %d, want 304", resp.StatusCode) |
| 151 | } |
| 152 | // A 304 carries what its 200 would have: the cache updates from it. |
| 153 | if got := resp.Header.Get("ETag"); got != homeV1 { |
| 154 | t.Errorf("304 ETag = %q, want the 200's %q", got, homeV1) |
| 155 | } |
| 156 | if got := resp.Header.Get("Cache-Control"); got != "no-cache" { |
| 157 | t.Errorf("304 Cache-Control = %q, want %q", got, "no-cache") |
| 158 | } |
| 159 | |
| 160 | // v2 changes the home page and leaves the stylesheet byte-identical. |
| 161 | shipSite(t, store, "site", "v2", map[string]string{"index.html": "home v2", "style.css": "body{}"}) |
| 162 | promote("v2") |
| 163 | resp, body := reqStatic(t, "GET", srv.URL, "/", map[string]string{"If-None-Match": homeV1}) |
| 164 | if resp.StatusCode != http.StatusOK || body != "home v2" { |
| 165 | t.Fatalf("revalidating a changed page after a deploy = %d %q, want 200 %q", resp.StatusCode, body, "home v2") |
| 166 | } |
| 167 | homeV2 := resp.Header.Get("ETag") |
| 168 | if homeV2 == homeV1 { |
| 169 | t.Fatalf("the changed page kept the tag %q", homeV1) |
| 170 | } |
| 171 | resp, _ = reqStatic(t, "GET", srv.URL, "/style.css", map[string]string{"If-None-Match": cssV1}) |
| 172 | if resp.StatusCode != http.StatusNotModified { |
| 173 | t.Fatalf("revalidating an UNCHANGED file after a deploy = %d, want 304", resp.StatusCode) |
| 174 | } |
| 175 | |
| 176 | promote("v1") // roll back |
| 177 | resp, body = reqStatic(t, "GET", srv.URL, "/", map[string]string{"If-None-Match": homeV2}) |
| 178 | if resp.StatusCode != http.StatusOK || body != "home v1" { |
| 179 | t.Fatalf("revalidating after a rollback = %d %q, want 200 %q", resp.StatusCode, body, "home v1") |
| 180 | } |
| 181 | if got := resp.Header.Get("ETag"); got != homeV1 { |
| 182 | t.Errorf("the rolled-back page's tag = %q, want its original %q", got, homeV1) |
| 183 | } |
| 184 | } |
| 185 | |
| 186 | // A browser whose copy predates the ETag holds only the old Last-Modified and |
| 187 | // revalidates on the date alone. Against an extraction mtime that answer is |
| 188 | // wrong after a rollback — the tree rolled back to was unpacked BEFORE the one |
| 189 | // the browser holds, so "modified since?" says no. With no date offered there |
| 190 | // is nothing to compare, and the browser gets the bytes. |
| 191 | func TestStaticDateOnlyRevalidationGetsTheBytes(t *testing.T) { |
| 192 | _, _, _, srv := staticFixture(t) |
| 193 | resp, body := reqStatic(t, "GET", srv.URL, "/", map[string]string{ |
| 194 | "If-Modified-Since": "Fri, 01 Jan 2100 00:00:00 GMT", |
| 195 | }) |
| 196 | if resp.StatusCode != http.StatusOK || body != "home v1" { |
| 197 | t.Fatalf("date-only revalidation = %d %q, want 200 %q", resp.StatusCode, body, "home v1") |
| 198 | } |
| 199 | } |
| 200 | |
| 201 | // A version NAME is not a fact about the bytes. A pruned version can be |
| 202 | // re-shipped under its old name with different content, and a hostname can |
| 203 | // move between two apps that both have a "v1" — in either case a tag made of |
| 204 | // the name would let a browser keep the old page on a 304. |
| 205 | func TestStaticSameVersionNameDifferentContent(t *testing.T) { |
| 206 | a := siteServer(t, "site", "v1", map[string]string{"index.html": "the first site"}) |
| 207 | b := siteServer(t, "other", "v1", map[string]string{"index.html": "a different site"}) |
| 208 | |
| 209 | respA, _ := reqStatic(t, "GET", a.URL, "/", nil) |
| 210 | tagA := respA.Header.Get("ETag") |
| 211 | if tagA == "" { |
| 212 | t.Fatal("no ETag from the first site") |
| 213 | } |
| 214 | resp, body := reqStatic(t, "GET", b.URL, "/", map[string]string{"If-None-Match": tagA}) |
| 215 | if resp.StatusCode != http.StatusOK || body != "a different site" { |
| 216 | t.Fatalf("revalidating one v1 against another app's v1 = %d %q, want 200 %q", resp.StatusCode, body, "a different site") |
| 217 | } |
| 218 | |
| 219 | // And identical bytes agree wherever they are served from — two edges |
| 220 | // unpacking the same release must hand out the same tag. |
| 221 | c := siteServer(t, "third", "v9", map[string]string{"index.html": "the first site"}) |
| 222 | resp, _ = reqStatic(t, "GET", c.URL, "/", map[string]string{"If-None-Match": tagA}) |
| 223 | if resp.StatusCode != http.StatusNotModified { |
| 224 | t.Fatalf("identical bytes from another handler = %d, want 304", resp.StatusCode) |
| 225 | } |
| 226 | } |
| 227 | |
| 228 | // The edge gzips a compressible response and, because gzipped bytes are a |
| 229 | // different representation, weakens a strong ETag as it does — but only on |
| 230 | // the 200. A 304 and a HEAD pass through untouched, so a strong tag would |
| 231 | // come back weak on the page and strong on its revalidation, and a cache may |
| 232 | // not update the one from the other. A tag that is weak from the start reads |
| 233 | // the same on all three. |
| 234 | func TestStaticTagIsTheSameCompressedRevalidatedAndOnHead(t *testing.T) { |
| 235 | page := "<html>" + strings.Repeat("a page big enough to be worth compressing. ", 200) + "</html>" |
| 236 | srv := siteServer(t, "site", "v1", map[string]string{"index.html": page}) |
| 237 | gz := map[string]string{"Accept-Encoding": "gzip"} |
| 238 | |
| 239 | resp, _ := reqStatic(t, "GET", srv.URL, "/", gz) |
| 240 | if resp.StatusCode != http.StatusOK || resp.Header.Get("Content-Encoding") != "gzip" { |
| 241 | t.Fatalf("GET / = %d, Content-Encoding %q; want a gzipped 200", resp.StatusCode, resp.Header.Get("Content-Encoding")) |
| 242 | } |
| 243 | tag := resp.Header.Get("ETag") |
| 244 | if !strings.HasPrefix(tag, `W/"`) { |
| 245 | t.Fatalf("gzipped 200 ETag = %q, want a weak tag", tag) |
| 246 | } |
| 247 | |
| 248 | resp, _ = reqStatic(t, "GET", srv.URL, "/", map[string]string{"Accept-Encoding": "gzip", "If-None-Match": tag}) |
| 249 | if resp.StatusCode != http.StatusNotModified { |
| 250 | t.Fatalf("revalidating the gzipped page = %d, want 304", resp.StatusCode) |
| 251 | } |
| 252 | if got := resp.Header.Get("ETag"); got != tag { |
| 253 | t.Errorf("304 ETag = %q, want the 200's %q", got, tag) |
| 254 | } |
| 255 | |
| 256 | resp, _ = reqStatic(t, "HEAD", srv.URL, "/", gz) |
| 257 | if got := resp.Header.Get("ETag"); got != tag { |
| 258 | t.Errorf("HEAD ETag = %q, want the GET's %q", got, tag) |
| 259 | } |
| 260 | // Asked for by name: with no Accept-Encoding of its own the Go client |
| 261 | // requests gzip and hides that it did. |
| 262 | resp, body := reqStatic(t, "GET", srv.URL, "/", map[string]string{"Accept-Encoding": "identity"}) |
| 263 | if resp.StatusCode != http.StatusOK || body != page || resp.Header.Get("Content-Encoding") != "" { |
| 264 | t.Fatalf("identity GET = %d, Content-Encoding %q, %d bytes; want the plain page", resp.StatusCode, resp.Header.Get("Content-Encoding"), len(body)) |
| 265 | } |
| 266 | if got := resp.Header.Get("ETag"); got != tag { |
| 267 | t.Errorf("identity GET ETag = %q, want the gzipped GET's %q", got, tag) |
| 268 | } |
| 269 | } |
| 270 | |
| 271 | // What http.ServeFile gave for free and this handler must keep now that it |
| 272 | // serves the file itself: the canonical redirect off /index.html, ranges, and |
| 273 | // the refusal of a path with a ".." element. |
| 274 | func TestStaticKeepsServeFileBehaviour(t *testing.T) { |
| 275 | _, _, _, srv := staticFixture(t) |
| 276 | |
| 277 | resp, _ := reqStatic(t, "GET", srv.URL, "/index.html?x=1", nil) |
| 278 | if resp.StatusCode != http.StatusMovedPermanently || resp.Header.Get("Location") != "./?x=1" { |
| 279 | t.Fatalf("GET /index.html = %d, Location %q; want 301 to ./?x=1", resp.StatusCode, resp.Header.Get("Location")) |
| 280 | } |
| 281 | |
| 282 | resp, body := reqStatic(t, "GET", srv.URL, "/", map[string]string{"Range": "bytes=0-3"}) |
| 283 | if resp.StatusCode != http.StatusPartialContent || body != "home" { |
| 284 | t.Fatalf("ranged GET = %d %q, want 206 %q", resp.StatusCode, body, "home") |
| 285 | } |
| 286 | // A weak tag cannot vouch for a byte range: If-Range falls back to the |
| 287 | // whole file, which is the safe answer and the documented cost. |
| 288 | tag := resp.Header.Get("ETag") |
| 289 | resp, body = reqStatic(t, "GET", srv.URL, "/", map[string]string{"Range": "bytes=0-3", "If-Range": tag}) |
| 290 | if resp.StatusCode != http.StatusOK || body != "home v1" { |
| 291 | t.Fatalf("If-Range with a weak tag = %d %q, want the full 200", resp.StatusCode, body) |
| 292 | } |
| 293 | |
| 294 | req, err := http.NewRequest("GET", srv.URL+"/", nil) |
| 295 | if err != nil { |
| 296 | t.Fatal(err) |
| 297 | } |
| 298 | req.Host = "site.example.com" |
| 299 | req.URL.Opaque = "/docs/../index.html" // sent as written, not cleaned by the client |
| 300 | resp2, err := http.DefaultClient.Do(req) |
| 301 | if err != nil { |
| 302 | t.Fatal(err) |
| 303 | } |
| 304 | resp2.Body.Close() |
| 305 | if resp2.StatusCode != http.StatusBadRequest { |
| 306 | t.Fatalf("a path with a .. element = %d, want 400", resp2.StatusCode) |
| 307 | } |
| 308 | } |
| 309 | |
| 310 | // The same bytes can be two representations. /about as an extensionless file |
| 311 | // is sniffed as text/plain; redeployed as about/index.html it is text/html, |
| 312 | // byte for byte the same. A tag made of the bytes alone would answer 304 and |
| 313 | // leave the browser rendering markup as text. |
| 314 | func TestStaticTagCoversTheContentType(t *testing.T) { |
| 315 | store, _, priv, srv := staticFixture(t) |
| 316 | const same = "Hello <em>world</em>" |
| 317 | shipSite(t, store, "site", "v2", map[string]string{"index.html": "home", "about": same}) |
| 318 | if err := release.Promote(context.Background(), store, priv, release.OpsAccount, "site", "canary/x", "v2"); err != nil { |
| 319 | t.Fatalf("Promote v2: %v", err) |
| 320 | } |
| 321 | resp, _ := reqStatic(t, "GET", srv.URL, "/about", nil) |
| 322 | plainTag, plainType := resp.Header.Get("ETag"), resp.Header.Get("Content-Type") |
| 323 | if resp.StatusCode != http.StatusOK || !strings.HasPrefix(plainType, "text/plain") { |
| 324 | t.Fatalf("GET /about = %d %q, want a text/plain 200", resp.StatusCode, plainType) |
| 325 | } |
| 326 | |
| 327 | shipSite(t, store, "site", "v3", map[string]string{"index.html": "home", "about/index.html": same}) |
| 328 | if err := release.Promote(context.Background(), store, priv, release.OpsAccount, "site", "canary/x", "v3"); err != nil { |
| 329 | t.Fatalf("Promote v3: %v", err) |
| 330 | } |
| 331 | resp, body := reqStatic(t, "GET", srv.URL, "/about", map[string]string{"If-None-Match": plainTag}) |
| 332 | if resp.StatusCode != http.StatusOK || body != same || !strings.HasPrefix(resp.Header.Get("Content-Type"), "text/html") { |
| 333 | t.Fatalf("revalidating /about after it became HTML = %d %q; want a text/html 200", resp.StatusCode, resp.Header.Get("Content-Type")) |
| 334 | } |
| 335 | } |
| 336 | |
| 337 | // openTagged writes body at dir/name and opens it the way serveTree does. |
| 338 | func openTagged(t *testing.T, dir, name, body string) (string, *os.File, os.FileInfo) { |
| 339 | t.Helper() |
| 340 | fp := filepath.Join(dir, name) |
| 341 | if err := os.WriteFile(fp, []byte(body), 0o644); err != nil { |
| 342 | t.Fatal(err) |
| 343 | } |
| 344 | f, err := os.Open(fp) |
| 345 | if err != nil { |
| 346 | t.Fatal(err) |
| 347 | } |
| 348 | t.Cleanup(func() { f.Close() }) |
| 349 | fi, err := f.Stat() |
| 350 | if err != nil { |
| 351 | t.Fatal(err) |
| 352 | } |
| 353 | return fp, f, fi |
| 354 | } |
| 355 | |
| 356 | // A cold file is hashed ONCE however many requests arrive for it together. |
| 357 | // Without that, a burst of requests against one large uncached asset — HEADs |
| 358 | // and one-byte ranges included — would each read and hash the whole file |
| 359 | // before sending a byte. |
| 360 | // |
| 361 | // The overlap is forced, not hoped for: the first hash is held until every |
| 362 | // other caller is known to be waiting on it. |
| 363 | func TestTagCacheHashesAColdFileOnceForOverlappingRequests(t *testing.T) { |
| 364 | c := newTagCache() |
| 365 | c.wait = time.Minute |
| 366 | held, release := make(chan struct{}), make(chan struct{}) |
| 367 | var heldOnce, releaseOnce sync.Once |
| 368 | c.hold = func() { heldOnce.Do(func() { close(held) }); <-release } |
| 369 | letGo := func() { releaseOnce.Do(func() { close(release) }) } |
| 370 | defer letGo() // a failure below must not strand the callers |
| 371 | dir := t.TempDir() |
| 372 | if err := os.WriteFile(filepath.Join(dir, "big.bin"), []byte(strings.Repeat("x", 4096)), 0o644); err != nil { |
| 373 | t.Fatal(err) |
| 374 | } |
| 375 | const callers = 12 |
| 376 | tags := make([]string, callers) |
| 377 | errs := make([]error, callers) |
| 378 | var wg sync.WaitGroup |
| 379 | call := func(i int) { |
| 380 | defer wg.Done() |
| 381 | fp := filepath.Join(dir, "big.bin") |
| 382 | f, err := os.Open(fp) |
| 383 | if err != nil { |
| 384 | errs[i] = err |
| 385 | return |
| 386 | } |
| 387 | defer f.Close() |
| 388 | fi, err := f.Stat() |
| 389 | if err != nil { |
| 390 | errs[i] = err |
| 391 | return |
| 392 | } |
| 393 | tags[i], _, errs[i] = c.tag(context.Background(), fp, f, fi) |
| 394 | } |
| 395 | wg.Add(1) |
| 396 | go call(0) |
| 397 | <-held // the first caller is inside its hash |
| 398 | for i := 1; i < callers; i++ { |
| 399 | wg.Add(1) |
| 400 | go call(i) |
| 401 | } |
| 402 | deadline := time.Now().Add(10 * time.Second) |
| 403 | for c.waiting.Load() != callers-1 { |
| 404 | if time.Now().After(deadline) { |
| 405 | t.Fatalf("%d of %d callers are waiting on the hash in flight", c.waiting.Load(), callers-1) |
| 406 | } |
| 407 | time.Sleep(time.Millisecond) |
| 408 | } |
| 409 | letGo() |
| 410 | wg.Wait() |
| 411 | for i := range tags { |
| 412 | if errs[i] != nil || tags[i] == "" || tags[i] != tags[0] { |
| 413 | t.Fatalf("caller %d = %q, %v; want every caller to get %q", i, tags[i], errs[i], tags[0]) |
| 414 | } |
| 415 | } |
| 416 | if got := c.hashed.Load(); got != 1 { |
| 417 | t.Fatalf("the file was hashed %d times for %d overlapping callers, want once", got, callers) |
| 418 | } |
| 419 | } |
| 420 | |
| 421 | // Large files hash a few at a time, and a request that cannot get a turn — |
| 422 | // or whose file is still being hashed for someone else — is not made to wait |
| 423 | // for it: past a short budget it is served without a tag. A small file never |
| 424 | // queues behind them at all. Otherwise four requests for four large cold |
| 425 | // assets would hold up the first load of every other static site on the box. |
| 426 | func TestTagCacheDoesNotQueueBehindLargeHashes(t *testing.T) { |
| 427 | c := newTagCache() |
| 428 | c.wait = 20 * time.Millisecond |
| 429 | dir := t.TempDir() |
| 430 | for i := 0; i < tagHashers; i++ { |
| 431 | c.sem <- struct{}{} // every hasher busy with someone's large asset |
| 432 | } |
| 433 | |
| 434 | fp, f, fi := openTagged(t, dir, "big.bin", strings.Repeat("x", tagInline+1)) |
| 435 | if _, _, err := c.tag(context.Background(), fp, f, fi); !errors.Is(err, errTagBusy) { |
| 436 | t.Fatalf("a large file with no free hasher = %v, want errTagBusy", err) |
| 437 | } |
| 438 | if n := len(c.flights); n != 0 { |
| 439 | t.Fatalf("%d flights left registered after giving up", n) |
| 440 | } |
| 441 | |
| 442 | fp, f, fi = openTagged(t, dir, "page.html", "<p>a small page</p>") |
| 443 | tag, ctype, err := c.tag(context.Background(), fp, f, fi) |
| 444 | if err != nil || tag == "" || !strings.HasPrefix(ctype, "text/html") { |
| 445 | t.Fatalf("a small file while every hasher is busy = %q %q %v; want it tagged at once", tag, ctype, err) |
| 446 | } |
| 447 | |
| 448 | // Once a hasher frees up the large file is tagged, and stays tagged. |
| 449 | // The budget is lifted first: at 20ms a stalled scheduler could find the |
| 450 | // timer and the free hasher ready together, and pick the timer. |
| 451 | <-c.sem |
| 452 | c.wait = time.Minute |
| 453 | fp, f, fi = openTagged(t, dir, "big.bin", strings.Repeat("x", tagInline+1)) |
| 454 | if tag, _, err := c.tag(context.Background(), fp, f, fi); err != nil || tag == "" { |
| 455 | t.Fatalf("a large file with a free hasher = %q, %v", tag, err) |
| 456 | } |
| 457 | } |
| 458 | |
| 459 | // The tag covers the Content-Type the response will carry, not the extension |
| 460 | // it was derived from: two boxes with different MIME tables type the same |
| 461 | // bytes differently, and a 304 from one must not confirm the other's page. |
| 462 | func TestTagCacheTagCoversTheResolvedContentType(t *testing.T) { |
| 463 | c := newTagCache() |
| 464 | const body = "# notes\n\nsome text\n" |
| 465 | // The MIME table is process-wide and an added type stays added, so each |
| 466 | // run takes an extension of its own (-count=N would otherwise find the |
| 467 | // last run's mapping in place of an unknown one). |
| 468 | ext := fmt.Sprintf(".zzcarlos%d", time.Now().UnixNano()) |
| 469 | fp, f, fi := openTagged(t, t.TempDir(), "notes"+ext, body) |
| 470 | sniffedTag, sniffedType, err := c.tag(context.Background(), fp, f, fi) |
| 471 | if err != nil || !strings.HasPrefix(sniffedType, "text/plain") { |
| 472 | t.Fatalf("an unknown extension = %q %v, want it sniffed as text/plain", sniffedType, err) |
| 473 | } |
| 474 | |
| 475 | // "Another box": the same bytes under the same name, with the type known. |
| 476 | if err := mime.AddExtensionType(ext, "text/markdown; charset=utf-8"); err != nil { |
| 477 | t.Fatal(err) |
| 478 | } |
| 479 | fp, f, fi = openTagged(t, t.TempDir(), "notes"+ext, body) |
| 480 | knownTag, knownType, err := c.tag(context.Background(), fp, f, fi) |
| 481 | if err != nil || !strings.HasPrefix(knownType, "text/markdown") { |
| 482 | t.Fatalf("a mapped extension = %q %v, want text/markdown", knownType, err) |
| 483 | } |
| 484 | if knownTag == sniffedTag { |
| 485 | t.Fatalf("the same bytes typed %q and %q share the tag %s", sniffedType, knownType, knownTag) |
| 486 | } |
| 487 | } |
| 488 | |
| 489 | // A remembered tag describes the file it was computed from, not the path. If |
| 490 | // the tree is replaced on disk under a live handler — cache maintenance, then |
| 491 | // another handler unpacking a re-shipped version of the same name — the old |
| 492 | // tag must not be handed out over the new bytes, nor a 304 against it. |
| 493 | func TestStaticTagDoesNotOutliveTheFileItDescribes(t *testing.T) { |
| 494 | srv, tree := siteServerTree(t, "site", "v1", map[string]string{"index.html": "home", "style.css": "body{}"}) |
| 495 | resp, _ := reqStatic(t, "GET", srv.URL, "/style.css", nil) |
| 496 | oldTag := resp.Header.Get("ETag") |
| 497 | if oldTag == "" { |
| 498 | t.Fatal("no ETag for /style.css") |
| 499 | } |
| 500 | |
| 501 | // Replace the file the way a fresh extraction would — a new file renamed |
| 502 | // into place — and make it the hardest case to notice: the same length |
| 503 | // and the same mtime, so only the file's identity gives it away. |
| 504 | fp := filepath.Join(tree, "style.css") |
| 505 | before, err := os.Stat(fp) |
| 506 | if err != nil { |
| 507 | t.Fatal(err) |
| 508 | } |
| 509 | if err := os.WriteFile(fp+".new", []byte("main{}"), 0o644); err != nil { |
| 510 | t.Fatal(err) |
| 511 | } |
| 512 | if err := os.Chtimes(fp+".new", before.ModTime(), before.ModTime()); err != nil { |
| 513 | t.Fatal(err) |
| 514 | } |
| 515 | if err := os.Rename(fp+".new", fp); err != nil { |
| 516 | t.Fatal(err) |
| 517 | } |
| 518 | |
| 519 | resp, body := reqStatic(t, "GET", srv.URL, "/style.css", map[string]string{"If-None-Match": oldTag}) |
| 520 | if resp.StatusCode != http.StatusOK || body != "main{}" { |
| 521 | t.Fatalf("revalidating against a replaced file = %d %q, want 200 with the new bytes", resp.StatusCode, body) |
| 522 | } |
| 523 | if got := resp.Header.Get("ETag"); got == "" || got == oldTag { |
| 524 | t.Fatalf("ETag over the replaced file = %q, want a new tag (old %q)", got, oldTag) |
| 525 | } |
| 526 | } |
| 527 | |
| 528 | // A file that is there and cannot be read is not a missing page. ServeFile |
| 529 | // said 403, and an operator reading the access log must still be able to tell |
| 530 | // a broken cache from a mistyped URL. |
| 531 | func TestStaticUnreadableFileIsNotA404(t *testing.T) { |
| 532 | if os.Geteuid() == 0 { |
| 533 | t.Skip("root reads through file modes") |
| 534 | } |
| 535 | srv, tree := siteServerTree(t, "site", "v1", map[string]string{"index.html": "home", "style.css": "body{}"}) |
| 536 | reqStatic(t, "GET", srv.URL, "/", nil) // extract the tree |
| 537 | fp := filepath.Join(tree, "style.css") |
| 538 | if err := os.Chmod(fp, 0); err != nil { |
| 539 | t.Fatal(err) |
| 540 | } |
| 541 | t.Cleanup(func() { os.Chmod(fp, 0o644) }) |
| 542 | resp, _ := reqStatic(t, "GET", srv.URL, "/style.css", nil) |
| 543 | if resp.StatusCode != http.StatusForbidden { |
| 544 | t.Fatalf("GET of an unreadable file = %d, want 403", resp.StatusCode) |
| 545 | } |
| 546 | } |
| 547 | |
| 548 | // failingFile reads like a file whose later blocks are bad: the first good |
| 549 | // bytes come back, then every read fails. Seeks work. |
| 550 | type failingFile struct { |
| 551 | data []byte |
| 552 | good int |
| 553 | pos int64 |
| 554 | } |
| 555 | |
| 556 | func (f *failingFile) Read(p []byte) (int, error) { |
| 557 | if f.pos >= int64(f.good) { |
| 558 | return 0, errors.New("input/output error") |
| 559 | } |
| 560 | n := copy(p, f.data[f.pos:f.good]) |
| 561 | f.pos += int64(n) |
| 562 | return n, nil |
| 563 | } |
| 564 | |
| 565 | func (f *failingFile) Seek(off int64, whence int) (int64, error) { |
| 566 | switch whence { |
| 567 | case io.SeekStart: |
| 568 | f.pos = off |
| 569 | case io.SeekCurrent: |
| 570 | f.pos += off |
| 571 | case io.SeekEnd: |
| 572 | f.pos = int64(len(f.data)) + off |
| 573 | } |
| 574 | return f.pos, nil |
| 575 | } |
| 576 | |
| 577 | // A hash that fails part-way must hand the file back at its start. The |
| 578 | // handler then serves it untagged, and net/http sniffs the Content-Type from |
| 579 | // wherever the file is positioned: left mid-file, an extensionless image |
| 580 | // would go out as text/plain. |
| 581 | func TestTagCacheRewindsTheFileWhenHashingFails(t *testing.T) { |
| 582 | c := newTagCache() |
| 583 | _, _, fi := openTagged(t, t.TempDir(), "picture", "placeholder") |
| 584 | for _, good := range []int{100, 2000} { // fails inside the sniffed prefix, and after it |
| 585 | f := &failingFile{data: make([]byte, 4096), good: good} |
| 586 | if _, _, err := c.tag(context.Background(), fmt.Sprintf("/nowhere/picture-%d", good), f, fi); err == nil { |
| 587 | t.Fatalf("hashing a file that fails after %d bytes reported no error", good) |
| 588 | } |
| 589 | if f.pos != 0 { |
| 590 | t.Fatalf("after a failed hash (%d good bytes) the file is at offset %d, want 0", good, f.pos) |
| 591 | } |
| 592 | } |
| 593 | if n := len(c.entries); n != 0 { |
| 594 | t.Fatalf("%d tags remembered from hashes that failed", n) |
| 595 | } |
| 596 | } |
| 597 | |
amadan