rastrillo / fiesta Public

Clone
git clone https://amadan.net/rastrillo/fiesta

Plain git — no account needed to clone.

Download

Download this file

1package www_test
2
3// The _headers grammar and matching of CARLOS's static edge, mirrored
4// from internal/edge/headers.go in carlos/platform (v0.31.1) so the
5// tests can refuse a file `carlos ship` would refuse and read the
6// headers a request gets. When the edge changes its grammar, this
7// changes with it.
8
9import (
10 "crypto/sha256"
11 "encoding/hex"
12 "errors"
13 "fmt"
14 "net/http"
15 "os"
16 "path"
17 "regexp"
18 "slices"
19 "strings"
20 "testing"
21
22 "amadan.net/rastrillo/fiesta"
23)
24
25// The edge's limits on one _headers file.
26const (
27 headersMaxBytes = 256 << 10
28 headersMaxPaths = 100
29 headersMaxLine = 2048
30)
31
32// headerRule is one path line of a _headers file and the headers under
33// it, in the order written.
34type headerRule struct {
35 line int
36 path string
37 rule // the path, read as a _redirects source is
38 set http.Header
39}
40
41// headerChecks are the headers the edge lets a site set, each with the
42// check its value must pass.
43var headerChecks = map[string]func(string) bool{
44 "Cache-Control": cacheDirectives,
45 "Access-Control-Allow-Origin": func(v string) bool { return v == "*" },
46 "Access-Control-Expose-Headers": headerNames,
47 "Cross-Origin-Resource-Policy": func(v string) bool {
48 return v == "same-origin" || v == "same-site" || v == "cross-origin"
49 },
50 "Timing-Allow-Origin": func(v string) bool { return v == "*" },
51}
52
53// parseHeaders parses a _headers file as `carlos ship` does: every
54// line it cannot use is an error, named by number.
55func parseHeaders(data []byte) ([]headerRule, error) {
56 if len(data) > headersMaxBytes {
57 return nil, fmt.Errorf("_headers is %d bytes; the limit is %d", len(data), headersMaxBytes)
58 }
59 var rules []headerRule
60 var errs []error
61 bad := func(n int, err error) { errs = append(errs, fmt.Errorf("line %d: %v", n, err)) }
62 var cur *headerRule
63 seenPath, paths := false, 0
64 finish := func() {
65 if cur != nil && len(cur.set) == 0 {
66 bad(cur.line, fmt.Errorf("path %q has no headers under it", cur.path))
67 } else if cur != nil {
68 rules = append(rules, *cur)
69 }
70 cur = nil
71 }
72 text := strings.TrimPrefix(string(data), "\ufeff")
73 for i, line := range strings.Split(text, "\n") {
74 n := i + 1
75 line = strings.TrimSuffix(line, "\r")
76 trimmed := strings.Trim(line, " \t")
77 if len(line) > headersMaxLine {
78 bad(n, fmt.Errorf("longer than %d characters", headersMaxLine))
79 if trimmed != "" && line[0] != ' ' && line[0] != '\t' && !strings.HasPrefix(trimmed, "#") {
80 finish()
81 seenPath = true
82 paths++
83 }
84 continue
85 }
86 if trimmed == "" || strings.HasPrefix(trimmed, "#") {
87 continue
88 }
89 if line[0] != ' ' && line[0] != '\t' {
90 finish()
91 seenPath = true
92 paths++
93 switch {
94 case strings.ContainsAny(trimmed, " \t"):
95 bad(n, fmt.Errorf("path line %q: want a path and nothing else", trimmed))
96 continue
97 case paths > headersMaxPaths:
98 bad(n, fmt.Errorf("more than %d path lines", headersMaxPaths))
99 continue
100 }
101 // The edge reads a path line with the parser it reads a
102 // _redirects source with, so the source half of a rule
103 // is that parser here.
104 r, err := parseRedirectLine(trimmed + " /")
105 if err != nil {
106 bad(n, err)
107 continue
108 }
109 cur = &headerRule{line: n, path: trimmed, rule: r, set: http.Header{}}
110 continue
111 }
112 if !seenPath {
113 bad(n, errors.New("header line before a path line"))
114 continue
115 }
116 if cur == nil {
117 continue
118 }
119 name, value, err := parseHeaderLine(trimmed)
120 if err != nil {
121 bad(n, err)
122 continue
123 }
124 cur.set.Set(name, value)
125 }
126 finish()
127 return rules, errors.Join(errs...)
128}
129
130// parseHeaderLine splits and checks one "Name: value" line as the edge
131// does, returning the canonical name and the trimmed value. The value
132// is held to visible ASCII and spaces before its header's check reads
133// it, as the edge holds it: a quoted string passes the Cache-Control
134// grammar whatever it holds, and a tab beside a comma is trimmed away
135// with the spaces, so without this a value the edge refuses would
136// parse here.
137func parseHeaderLine(line string) (name, value string, err error) {
138 if strings.HasPrefix(line, "!") {
139 return "", "", fmt.Errorf("%q: removing a header with ! is not supported", line)
140 }
141 n, v, ok := strings.Cut(line, ":")
142 n = strings.Trim(n, " \t")
143 if !ok || n == "" {
144 return "", "", fmt.Errorf("%q: want a header line like \"Name: value\"", line)
145 }
146 name = http.CanonicalHeaderKey(n)
147 check, ok := headerChecks[name]
148 if !ok {
149 return "", "", fmt.Errorf("header %q is not supported", name)
150 }
151 value = strings.Trim(v, " \t")
152 if value == "" {
153 return "", "", fmt.Errorf("%s: the value is empty", name)
154 }
155 for i := 0; i < len(value); i++ {
156 if value[i] < ' ' || value[i] > '~' {
157 return "", "", fmt.Errorf("%s: the value may hold only visible ASCII and spaces", name)
158 }
159 }
160 if !check(value) {
161 return "", "", fmt.Errorf("%s: the edge refuses %q", name, value)
162 }
163 return name, value, nil
164}
165
166// isToken reports whether s is an HTTP token (RFC 9110).
167func isToken(s string) bool {
168 if s == "" {
169 return false
170 }
171 for i := 0; i < len(s); i++ {
172 c := s[i]
173 if !(c >= '0' && c <= '9' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || strings.IndexByte("!#$%&'*+-.^_`|~", c) >= 0) {
174 return false
175 }
176 }
177 return true
178}
179
180// splitOutsideQuotes splits v at the commas outside a quoted string,
181// as the edge does, so no-cache="X-One, X-Two" stays one directive.
182// An unterminated quote leaves the rest of v in one piece, which
183// cacheDirectives then refuses.
184func splitOutsideQuotes(v string) []string {
185 var parts []string
186 start, quoted := 0, false
187 for i := 0; i < len(v); i++ {
188 switch v[i] {
189 case '"':
190 quoted = !quoted
191 case ',':
192 if !quoted {
193 parts = append(parts, v[start:i])
194 start = i + 1
195 }
196 }
197 }
198 return append(parts, v[start:])
199}
200
201// cacheDirectives: a comma-separated list of directives, each a token
202// or token=value, the value a token or a quoted string, split only at
203// commas outside quotes.
204func cacheDirectives(v string) bool {
205 for _, d := range splitOutsideQuotes(v) {
206 name, val, hasVal := strings.Cut(strings.Trim(d, " \t"), "=")
207 quoted := len(val) >= 2 && val[0] == '"' && val[len(val)-1] == '"' && !strings.ContainsAny(val[1:len(val)-1], `"\`)
208 if !isToken(name) || hasVal && !isToken(val) && !quoted {
209 return false
210 }
211 }
212 return true
213}
214
215// headerNames: a comma-separated list of header names.
216func headerNames(v string) bool {
217 for _, n := range strings.Split(v, ",") {
218 if !isToken(strings.Trim(n, " \t")) {
219 return false
220 }
221 }
222 return true
223}
224
225// lookupHeaders is what the edge adds to a file served for host at the
226// cleaned path p, the path the browser asked for: every matching
227// rule's headers in file order, the last to name a header winning.
228func lookupHeaders(rules []headerRule, host, p string) http.Header {
229 out := http.Header{}
230 segs := segments(p)
231 for i := range rules {
232 if _, ok := rules[i].match(host, segs); !ok {
233 continue
234 }
235 for name := range rules[i].set {
236 out.Set(name, rules[i].set.Get(name))
237 }
238 }
239 return out
240}
241
242// The headers fiesta site sets, as the edge reads them.
243const (
244 yearLong = "public, max-age=31536000, immutable"
245 fiveMinute = "public, max-age=300"
246)
247
248// crossOrigin are the headers that let a page on another site fetch a
249// release's files, load them as modules and pin them by hash.
250var crossOrigin = http.Header{
251 "Access-Control-Allow-Origin": {"*"},
252 "Cross-Origin-Resource-Policy": {"cross-origin"},
253}
254
255// withCache is h with a Cache-Control of cc added, h left alone.
256func withCache(h http.Header, cc string) http.Header {
257 out := h.Clone()
258 if out == nil {
259 out = http.Header{}
260 }
261 out.Set("Cache-Control", cc)
262 return out
263}
264
265// versionDir is a release directory's name, X.Y.Z, as site.Build reads it.
266var versionDir = regexp.MustCompile(`^(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)$`)
267
268// hashedAsset is a design-system asset's hashed name: its stem, sixteen
269// hex digits and its extension.
270var hashedAsset = regexp.MustCompile(`^design-system/assets/.*\.([0-9a-f]{16})\.[^./]+$`)
271
272// fingerprinted reports whether the file at name is one of the design
273// system's hashed assets: its name carries the first sixteen hex
274// digits of its bytes' sha256, so other bytes would be another name.
275// It is worked out from the built tree alone, not from the gallery's
276// own list, so the two are compared rather than one copied.
277func fingerprinted(name string, data []byte) bool {
278 m := hashedAsset.FindStringSubmatch(name)
279 if m == nil {
280 return false
281 }
282 sum := sha256.Sum256(data)
283 return hex.EncodeToString(sum[:8]) == m[1]
284}
285
286// versionLess orders X.Y.Z names numerically.
287func versionLess(a, b string) int {
288 var x, y [3]int
289 fmt.Sscanf(a, "%d.%d.%d", &x[0], &x[1], &x[2])
290 fmt.Sscanf(b, "%d.%d.%d", &y[0], &y[1], &y[2])
291 for i := range x {
292 if x[i] != y[i] {
293 return x[i] - y[i]
294 }
295 }
296 return 0
297}
298
299// TestTheSitesHeaders reads the built site's _headers as the edge
300// does, twice: as every build but the one that ships makes it, with no
301// release published, and as fiesta site -published makes it, here with
302// every release but the current one published, so one tree holds both
303// kinds. Each is held to what the site needs:
304//
305// - exactly its rules: /M/* for each major, five minutes, since /M/
306// moves to each new M.x; /X.Y.Z/* for each release not published,
307// five minutes too, since a candidate may be rebuilt under its
308// name; /X.Y.Z/* for each published release, a year and immutable,
309// since its tag holds it; all with the cross-origin headers; and
310// each of the design system's hashed assets, a year and immutable,
311// alone. In that order, general before specific.
312// - every file the browser can be served immutable is a published
313// release's or a hashed asset, and every one of those is
314// immutable; every other page gets no Cache-Control from the file,
315// so the edge's no-cache stays.
316// - every rule's path matches at least one built file.
317// - the file is one `carlos ship` packs, under the edge's limits.
318func TestTheSitesHeaders(t *testing.T) {
319 entries, err := os.ReadDir("../../releases")
320 if err != nil {
321 t.Fatal(err)
322 }
323 var older []string
324 for _, e := range entries {
325 if versionDir.MatchString(e.Name()) && e.Name() != fiesta.Version {
326 older = append(older, e.Name())
327 }
328 }
329 if len(older) == 0 {
330 t.Fatalf("releases/ holds no release but %s, so the publish build would have nothing published to check", fiesta.Version)
331 }
332 pages := sitePages(t)
333 for _, build := range []struct {
334 name string
335 published []string
336 }{
337 {"nothing published", nil},
338 {"all but the current release published", older},
339 } {
340 t.Run(build.name, func(t *testing.T) {
341 siteHeadersHold(t, buildSite(t, pages, build.published), build.published)
342 })
343 }
344}
345
346// siteHeadersHold is TestTheSitesHeaders' checks on one built tree,
347// whose published releases are published.
348func siteHeadersHold(t *testing.T, tree map[string][]byte, published []string) {
349 data, ok := tree["_headers"]
350 if !ok {
351 t.Fatal("the site has no _headers")
352 }
353 rules, err := parseHeaders(data)
354 if err != nil {
355 t.Fatalf("carlos ship would refuse _headers: %v", err)
356 }
357 // releaseCache is what a release's files must get: a year only
358 // once it is published.
359 releaseCache := func(v string) string {
360 if slices.Contains(published, v) {
361 return yearLong
362 }
363 return fiveMinute
364 }
365
366 var versions []string
367 majors := map[string]bool{}
368 var hashed []string
369 for name, body := range tree {
370 first, _, _ := strings.Cut(name, "/")
371 if versionDir.MatchString(first) && !slices.Contains(versions, first) {
372 versions = append(versions, first)
373 major, _, _ := strings.Cut(first, ".")
374 majors[major] = true
375 }
376 if fingerprinted(name, body) {
377 hashed = append(hashed, "/"+name)
378 }
379 }
380 slices.SortFunc(versions, versionLess)
381 slices.Sort(hashed)
382 if !slices.Contains(versions, fiesta.Version) || !slices.Contains(versions, "1.0.0") || len(hashed) == 0 {
383 t.Fatalf("the site holds releases %v and %d hashed assets; the checks below need 1.0.0, the current release and the gallery's assets", versions, len(hashed))
384 }
385 majorList := make([]string, 0, len(majors))
386 for m := range majors {
387 majorList = append(majorList, m)
388 }
389 slices.SortFunc(majorList, func(a, b string) int { return versionLess(a+".0.0", b+".0.0") })
390
391 t.Run("exactly its rules", func(t *testing.T) {
392 type want struct {
393 path string
394 set http.Header
395 }
396 var wants []want
397 for _, m := range majorList {
398 wants = append(wants, want{"/" + m + "/*", withCache(crossOrigin, fiveMinute)})
399 }
400 for _, cache := range []string{fiveMinute, yearLong} {
401 for _, v := range versions {
402 if releaseCache(v) == cache {
403 wants = append(wants, want{"/" + v + "/*", withCache(crossOrigin, cache)})
404 }
405 }
406 }
407 for _, p := range hashed {
408 wants = append(wants, want{p, withCache(nil, yearLong)})
409 }
410 for i := 0; i < max(len(rules), len(wants)); i++ {
411 switch {
412 case i >= len(rules):
413 t.Errorf("_headers has no rule for %s", wants[i].path)
414 case i >= len(wants):
415 t.Errorf("_headers line %d: %s is a rule the site does not need", rules[i].line, rules[i].path)
416 case rules[i].path != wants[i].path || fmt.Sprint(rules[i].set) != fmt.Sprint(wants[i].set):
417 t.Errorf("_headers line %d: %s %v; want %s %v", rules[i].line, rules[i].path, rules[i].set, wants[i].path, wants[i].set)
418 }
419 }
420 t.Logf("_headers: %d majors, %d releases (%d published), %d hashed assets", len(majorList), len(versions), len(published), len(hashed))
421 })
422
423 t.Run("immutable is a published release or a hashed asset", func(t *testing.T) {
424 for name, body := range tree {
425 if name == "_redirects" || name == "_headers" {
426 continue
427 }
428 first, _, _ := strings.Cut(name, "/")
429 got := lookupHeaders(rules, "", path.Clean("/"+name))
430 cc := got.Get("Cache-Control")
431 switch {
432 case versionDir.MatchString(first):
433 if want := releaseCache(first); cc != want {
434 t.Errorf("/%s gets Cache-Control %q; want %q", name, cc, want)
435 }
436 case fingerprinted(name, body):
437 if cc != yearLong {
438 t.Errorf("/%s gets Cache-Control %q; it never changes, so want %q", name, cc, yearLong)
439 }
440 case len(got) != 0:
441 t.Errorf("/%s gets %v; it can change under its name, so it must keep the edge's defaults", name, got)
442 }
443 }
444 })
445
446 t.Run("every rule matches a built file", func(t *testing.T) {
447 for _, r := range rules {
448 found := false
449 for name := range tree {
450 if _, ok := r.match("", segments(path.Clean("/"+name))); ok && name != "_headers" && name != "_redirects" {
451 found = true
452 break
453 }
454 }
455 // A major's files are its newest release's, through the
456 // rewrite, not files at /M/ itself.
457 if !found && strings.HasSuffix(r.path, "/*") {
458 m := strings.Trim(strings.TrimSuffix(r.path, "*"), "/")
459 for _, v := range versions {
460 if strings.HasPrefix(v, m+".") {
461 found = true
462 }
463 }
464 }
465 if !found {
466 t.Errorf("_headers line %d: %s matches no file the site serves", r.line, r.path)
467 }
468 }
469 })
470
471 t.Run("served as the edge serves them", func(t *testing.T) {
472 h := newSiteHandler(t, tree)
473 major, _, _ := strings.Cut(fiesta.Version, ".")
474 for _, c := range []struct {
475 path, cache, origin string
476 }{
477 // Rules match before the rewrite, so /1/ is /1/*'s, not
478 // its release's.
479 {"/" + major + "/fiesta.css", fiveMinute, "*"},
480 {"/" + fiesta.Version + "/fiesta.css", releaseCache(fiesta.Version), "*"},
481 {"/1.0.0/i18n/es.json", releaseCache("1.0.0"), "*"},
482 {hashed[0], yearLong, ""},
483 {"/design-system/", "no-cache", ""},
484 {"/", "no-cache", ""},
485 } {
486 res := serve(h, c.path)
487 if res.Code != http.StatusOK {
488 t.Errorf("%s answers %d", c.path, res.Code)
489 continue
490 }
491 if cc := res.Header().Get("Cache-Control"); cc != c.cache {
492 t.Errorf("%s gets Cache-Control %q; want %q", c.path, cc, c.cache)
493 }
494 if o := res.Header().Get("Access-Control-Allow-Origin"); o != c.origin {
495 t.Errorf("%s gets Access-Control-Allow-Origin %q; want %q", c.path, o, c.origin)
496 }
497 }
498 if res := serve(h, "/_headers"); res.Code != http.StatusNotFound {
499 t.Errorf("/_headers answers %d; the edge never serves it", res.Code)
500 }
501 })
502
503 t.Run("within the edge's limits", func(t *testing.T) {
504 longest := 0
505 for _, l := range strings.Split(string(data), "\n") {
506 longest = max(longest, len(l))
507 }
508 if len(rules) > 90 {
509 t.Errorf("_headers names %d paths; the build stops at 90, ten short of the edge's %d", len(rules), headersMaxPaths)
510 }
511 t.Logf("_headers: %d paths, %d bytes, longest line %d characters", len(rules), len(data), longest)
512 })
513}
514
515// TestServeSiteSendsHeadersLikeTheEdge holds the test server's headers
516// to the edge's: a file served with a 200 gets no-cache, then every
517// _headers rule that matches the address asked for, before any
518// rewrite, in file order, the last to set a header winning; ":name"
519// matches one segment; a redirect, a rewrite's 404 and a missing file
520// get none of them; and the file itself is never served. A test of
521// the test double, not of the site.
522func TestServeSiteSendsHeadersLikeTheEdge(t *testing.T) {
523 h := newSiteHandler(t, map[string][]byte{
524 "_redirects": []byte("/1/* /1.0.0/:splat 200\n" +
525 "/old /index.html 301\n" +
526 "/gone/* /index.html 404\n"),
527 "_headers": []byte("\ufeff# general first\n" +
528 "/*\n" +
529 " timing-allow-origin: *\n" +
530 "/1/*\n" +
531 " Cache-Control: public, max-age=300\n" +
532 "/1.0.0/*\n" +
533 " Cache-Control: public, max-age=31536000, immutable\n" +
534 " Access-Control-Allow-Origin: *\n" +
535 "\n" +
536 "/1.0.0/:file\n" +
537 " Cache-Control: no-transform\n" +
538 "/old\n" +
539 " Cache-Control: public, max-age=60\n" +
540 "/gone/*\n" +
541 " Cache-Control: public, max-age=60\n"),
542 "index.html": []byte("root"),
543 "1.0.0/fiesta.css": []byte("newest"),
544 "1.0.0/i18n/es.js": []byte("strings"),
545 })
546 for _, c := range []struct {
547 path string
548 status int
549 cache, origin string
550 timing string
551 }{
552 {"/", 200, "no-cache", "", "*"},
553 {"/1/fiesta.css", 200, "public, max-age=300", "", "*"},
554 {"/1.0.0/fiesta.css", 200, "no-transform", "*", "*"},
555 {"/1.0.0/i18n/es.js", 200, "public, max-age=31536000, immutable", "*", "*"},
556 {"/old", 301, "", "", ""},
557 {"/gone/x", 404, "", "", ""},
558 {"/nope", 404, "", "", ""},
559 {"/_headers", 404, "", "", ""},
560 } {
561 res := serve(h, c.path)
562 got := []string{res.Header().Get("Cache-Control"), res.Header().Get("Access-Control-Allow-Origin"), res.Header().Get("Timing-Allow-Origin")}
563 if want := []string{c.cache, c.origin, c.timing}; res.Code != c.status || !slices.Equal(got, want) {
564 t.Errorf("GET %s: %d, Cache-Control %q, Allow-Origin %q, Timing-Allow-Origin %q; want %d %q", c.path, res.Code, got[0], got[1], got[2], c.status, want)
565 }
566 }
567}
568
569// TestServeSiteRefusesAHeadersFileCarlosWouldRefuse: the test server
570// reads _headers with CARLOS's grammar, so a file `carlos ship` would
571// refuse is refused here too, and one it packs is read.
572func TestServeSiteRefusesAHeadersFileCarlosWouldRefuse(t *testing.T) {
573 var many strings.Builder
574 for i := 0; i <= headersMaxPaths; i++ {
575 fmt.Fprintf(&many, "/p%d\n Timing-Allow-Origin: *\n", i)
576 }
577 for _, file := range []string{
578 " Cache-Control: no-cache\n",
579 "/a b\n Cache-Control: no-cache\n",
580 "/a\n",
581 "/a\n Set-Cookie: a=b\n",
582 "/a\n Strict-Transport-Security: max-age=1\n",
583 "/a\n Access-Control-Allow-Origin: https://example.com\n",
584 "/a\n Cross-Origin-Resource-Policy: anywhere\n",
585 "/a\n Cache-Control: public max-age=1\n",
586 "/a\n Cache-Control:\n",
587 "/a\n ! Cache-Control\n",
588 "/a/*/b\n Cache-Control: no-cache\n",
589 "/a\n Cache-Control: " + strings.Repeat("x", headersMaxLine) + "\n",
590 many.String(),
591 // Visible ASCII only, checked before the directives are read:
592 // a quoted string passes the directive grammar whatever it
593 // holds, and a tab is trimmed away around a directive.
594 "/a\n Cache-Control: private=\"Ünïcode\"\n",
595 "/a\n Cache-Control: public,\tmax-age=1\n",
596 } {
597 if _, err := parseHeaders([]byte(file)); err == nil {
598 t.Errorf("%.60q parsed; CARLOS refuses it", file)
599 }
600 }
601 for _, file := range []string{
602 "# a comment alone\n",
603 "/a\n cache-control: no-cache\n",
604 "/a/:x/*\n\tAccess-Control-Expose-Headers: ETag, Content-Length\n",
605 "https://fiesta.rastrillo.org/a\n Cache-Control: max-age=1, stale-if-error=\"60\"\n",
606 // A comma inside a quoted string does not end the directive.
607 "/a\n Cache-Control: no-cache=\"X-One, X-Two\"\n",
608 } {
609 if _, err := parseHeaders([]byte(file)); err != nil {
610 t.Errorf("%q was refused (%v); CARLOS accepts it", file, err)
611 }
612 }
613}
614