| 1 | package www_test |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "fmt" |
| 6 | "io/fs" |
| 7 | "maps" |
| 8 | "net/http" |
| 9 | "net/http/httptest" |
| 10 | "net/url" |
| 11 | "os" |
| 12 | "regexp" |
| 13 | "slices" |
| 14 | "sort" |
| 15 | "strings" |
| 16 | "testing" |
| 17 | "testing/fstest" |
| 18 | |
| 19 | xhtml "golang.org/x/net/html" |
| 20 | |
| 21 | "amadan.net/rastrillo/fiesta" |
| 22 | "amadan.net/rastrillo/fiesta/internal/build" |
| 23 | "amadan.net/rastrillo/fiesta/internal/site" |
| 24 | "amadan.net/rastrillo/fiesta/internal/www" |
| 25 | ) |
| 26 | |
| 27 | // outsideURLs are the URLs the site links on purpose that are not its |
| 28 | // own: the top bar's, the footer's, and the design system's one |
| 29 | // outbound link, lucide.dev, where its icons come from. Exactly these, |
| 30 | // as written, so a new outside link is a decision someone takes here |
| 31 | // rather than a line that slips in. The site's own absolute URLs, the hreflang |
| 32 | // links and the install text, start with www.Origin and are resolved |
| 33 | // as pages of the site instead, so a wrong path under the origin is |
| 34 | // caught; the origin itself is never on this list. |
| 35 | var outsideURLs = []string{ |
| 36 | "https://rastrillo.org", |
| 37 | "https://rastrillo.org/docs/", |
| 38 | "https://amadan.net/rastrillo/fiesta", |
| 39 | "https://amadan.net/rastrillo/rastrillo", |
| 40 | "https://carlosframework.com", |
| 41 | "https://carlosframework.com/platform/", |
| 42 | "https://carlosframework.com/start/", |
| 43 | "https://carlosframework.com/letters/", |
| 44 | "https://carlosframework.com/trust/", |
| 45 | "https://carloku.com", |
| 46 | "https://11factor.org", |
| 47 | "https://lucide.dev", |
| 48 | } |
| 49 | |
| 50 | // textURL is an absolute URL inside text: the install box's <code> |
| 51 | // shows a whole link tag as text, and the URL in it has to reach the |
| 52 | // stylesheet like any other. |
| 53 | var textURL = regexp.MustCompile(`https?://[^\s"'<>]+`) |
| 54 | |
| 55 | // reference is one URL a page carries: where it was found and what it |
| 56 | // says. |
| 57 | type reference struct { |
| 58 | page string // the tree path of the page, index.html or ar/index.html |
| 59 | where string // the element and attribute, "a href", or "code text" |
| 60 | raw string // the URL as written |
| 61 | } |
| 62 | |
| 63 | // referencesOf reads every href, src and srcset on a page, whatever |
| 64 | // the element (a, link, script, img, iframe, use...), and every URL in |
| 65 | // the text of a translate="no" code element, the one place a URL is |
| 66 | // shown rather than linked. |
| 67 | func referencesOf(page string, doc *xhtml.Node) []reference { |
| 68 | var out []reference |
| 69 | walk(doc, func(n *xhtml.Node) { |
| 70 | for _, a := range n.Attr { |
| 71 | switch a.Key { |
| 72 | case "href", "src": |
| 73 | out = append(out, reference{page, n.Data + " " + a.Key, a.Val}) |
| 74 | case "srcset": |
| 75 | // Comma-separated candidates, each a URL and an |
| 76 | // optional descriptor. |
| 77 | for _, c := range strings.Split(a.Val, ",") { |
| 78 | if f := strings.Fields(c); len(f) > 0 { |
| 79 | out = append(out, reference{page, n.Data + " srcset", f[0]}) |
| 80 | } |
| 81 | } |
| 82 | } |
| 83 | } |
| 84 | if n.Data == "code" && attr(n, "translate") == "no" { |
| 85 | for _, u := range textURL.FindAllString(text(n), -1) { |
| 86 | out = append(out, reference{page, "code text", u}) |
| 87 | } |
| 88 | } |
| 89 | }) |
| 90 | return out |
| 91 | } |
| 92 | |
| 93 | // served is what the edge answers for one path: the status, and the |
| 94 | // ids on the page if it is one, read once. |
| 95 | type served struct { |
| 96 | status int |
| 97 | html bool |
| 98 | ids map[string]bool |
| 99 | } |
| 100 | |
| 101 | // resolver answers for the built site the way the edge does, through |
| 102 | // serveSite's own handler, so a directory index, the _redirects |
| 103 | // rewrite and a real file count and nothing else does. |
| 104 | type resolver struct { |
| 105 | h *siteHandler |
| 106 | cache map[string]*served |
| 107 | } |
| 108 | |
| 109 | func newResolver(t *testing.T, tree map[string][]byte) *resolver { |
| 110 | t.Helper() |
| 111 | return &resolver{h: newSiteHandler(t, tree), cache: map[string]*served{}} |
| 112 | } |
| 113 | |
| 114 | // get serves p through the handler, once per path. |
| 115 | func (r *resolver) get(p string) *served { |
| 116 | if s, ok := r.cache[p]; ok { |
| 117 | return s |
| 118 | } |
| 119 | rec := httptest.NewRecorder() |
| 120 | r.h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, p, nil)) |
| 121 | s := &served{status: rec.Code} |
| 122 | if s.status == http.StatusOK && strings.HasPrefix(rec.Header().Get("Content-Type"), "text/html") { |
| 123 | s.html = true |
| 124 | s.ids = map[string]bool{} |
| 125 | if doc, err := xhtml.Parse(rec.Body); err == nil { |
| 126 | walk(doc, func(n *xhtml.Node) { |
| 127 | if id := attr(n, "id"); id != "" { |
| 128 | s.ids[id] = true |
| 129 | } |
| 130 | }) |
| 131 | } |
| 132 | } |
| 133 | r.cache[p] = s |
| 134 | return s |
| 135 | } |
| 136 | |
| 137 | // check reports what is wrong with one reference, or "" when it |
| 138 | // resolves: an outside URL not on the list, a path the edge would not |
| 139 | // serve, or a fragment with no id on the page it names. A bare # is a |
| 140 | // link to nowhere on purpose, as in a sample, and is left alone. |
| 141 | func (r *resolver) check(ref reference) string { |
| 142 | if ref.raw == "#" { |
| 143 | return "" |
| 144 | } |
| 145 | raw := ref.raw |
| 146 | u, err := url.Parse(raw) |
| 147 | if err != nil { |
| 148 | return fmt.Sprintf("%s: %s %q does not parse: %v", ref.page, ref.where, raw, err) |
| 149 | } |
| 150 | if u.Scheme != "" || u.Host != "" { |
| 151 | if raw == www.Origin || strings.HasPrefix(raw, www.Origin+"/") { |
| 152 | // The site's own, written in full: resolve the rest as a |
| 153 | // root-relative path of this very tree. |
| 154 | raw = strings.TrimPrefix(raw, www.Origin) |
| 155 | if raw == "" { |
| 156 | raw = "/" |
| 157 | } |
| 158 | if u, err = url.Parse(raw); err != nil { |
| 159 | return fmt.Sprintf("%s: %s %q does not parse: %v", ref.page, ref.where, ref.raw, err) |
| 160 | } |
| 161 | } else { |
| 162 | for _, ok := range outsideURLs { |
| 163 | if raw == ok { |
| 164 | return "" |
| 165 | } |
| 166 | } |
| 167 | return fmt.Sprintf("%s: %s links %s, which leaves the site and is not among the outside URLs the site links on purpose", ref.page, ref.where, raw) |
| 168 | } |
| 169 | } |
| 170 | base := &url.URL{Path: "/" + ref.page} |
| 171 | target := base.ResolveReference(u) |
| 172 | s := r.get(target.Path) |
| 173 | if s.status != http.StatusOK { |
| 174 | return fmt.Sprintf("%s: %s %s resolves to %s, which the edge answers with a %d", ref.page, ref.where, ref.raw, target.Path, s.status) |
| 175 | } |
| 176 | if target.Fragment == "" { |
| 177 | return "" |
| 178 | } |
| 179 | if !s.html { |
| 180 | return fmt.Sprintf("%s: %s %s names a fragment on %s, which is not a page", ref.page, ref.where, ref.raw, target.Path) |
| 181 | } |
| 182 | if !s.ids[target.Fragment] { |
| 183 | return fmt.Sprintf("%s: %s %s names #%s, and %s has no element with that id", ref.page, ref.where, ref.raw, target.Fragment, target.Path) |
| 184 | } |
| 185 | return "" |
| 186 | } |
| 187 | |
| 188 | // linkProblems checks every reference on every HTML file of a built |
| 189 | // site and returns what is wrong, one line each, sorted, with the |
| 190 | // pages it read. |
| 191 | func linkProblems(t *testing.T, tree map[string][]byte) (problems []string, pages []string) { |
| 192 | t.Helper() |
| 193 | r := newResolver(t, tree) |
| 194 | for name, data := range tree { |
| 195 | if !strings.HasSuffix(name, ".html") { |
| 196 | continue |
| 197 | } |
| 198 | pages = append(pages, name) |
| 199 | for _, ref := range referencesOf(name, parse(t, data)) { |
| 200 | if p := r.check(ref); p != "" { |
| 201 | problems = append(problems, p) |
| 202 | } |
| 203 | } |
| 204 | } |
| 205 | sort.Strings(problems) |
| 206 | sort.Strings(pages) |
| 207 | return problems, pages |
| 208 | } |
| 209 | |
| 210 | // unpublished reports whether the source is ahead of releases/: there |
| 211 | // is no releases/<version>/, or it holds other files than dist, the |
| 212 | // release the source builds. That is the stretch between a change to |
| 213 | // the source and the release that carries it, and the stretch the |
| 214 | // design system's pages spend linking files of the release to come. |
| 215 | func unpublished(releases fs.FS, version string, dist map[string][]byte) (bool, error) { |
| 216 | if _, err := fs.Stat(releases, version); err != nil { |
| 217 | return true, nil |
| 218 | } |
| 219 | dir, err := fs.Sub(releases, version) |
| 220 | if err != nil { |
| 221 | return false, err |
| 222 | } |
| 223 | published, err := site.Release(dir) |
| 224 | if err != nil { |
| 225 | return false, err |
| 226 | } |
| 227 | return !maps.EqualFunc(published, dist, bytes.Equal), nil |
| 228 | } |
| 229 | |
| 230 | // servingSource is tree as the edge will serve it once dist is |
| 231 | // released: dist sits beside the published releases, and the rewrite |
| 232 | // for version's major sends /M/ to it rather than to the newest |
| 233 | // published release. It sits under version/ when that is free, where |
| 234 | // fiesta release will put it, and under unreleased/ when version is |
| 235 | // published already and the source has moved past it, so a pinned |
| 236 | // /version/ link still reaches the bytes that version shipped. |
| 237 | func servingSource(t *testing.T, tree map[string][]byte, version string, dist map[string][]byte) map[string][]byte { |
| 238 | t.Helper() |
| 239 | out := maps.Clone(tree) |
| 240 | dir := version |
| 241 | for name := range tree { |
| 242 | if strings.HasPrefix(name, version+"/") { |
| 243 | dir = "unreleased" |
| 244 | break |
| 245 | } |
| 246 | } |
| 247 | for name, data := range dist { |
| 248 | out[dir+"/"+name] = data |
| 249 | } |
| 250 | major, _, _ := strings.Cut(version, ".") |
| 251 | prefix := "/" + major + "/* " |
| 252 | rule := prefix + "/" + dir + "/:splat 200" |
| 253 | var lines []string |
| 254 | found := false |
| 255 | for _, line := range strings.Split(strings.TrimSuffix(string(tree["_redirects"]), "\n"), "\n") { |
| 256 | if strings.HasPrefix(line, prefix) { |
| 257 | line, found = rule, true |
| 258 | } |
| 259 | lines = append(lines, line) |
| 260 | } |
| 261 | if !found { |
| 262 | // A new major's first release: its rule goes first, as |
| 263 | // site.Build writes the newest major's. |
| 264 | lines = append([]string{rule}, lines...) |
| 265 | } |
| 266 | out["_redirects"] = []byte(strings.Join(lines, "\n") + "\n") |
| 267 | return out |
| 268 | } |
| 269 | |
| 270 | // publishedOnlyEnv turns off the unpublished fallback below: the |
| 271 | // ship check, make site-links-published, sets it on the exported tree |
| 272 | // about to go out, where a /1/ link the newest published release |
| 273 | // cannot answer is a 404 the day the site ships, whatever the source |
| 274 | // would build. The fallback alone would pass such a link: today |
| 275 | // releases/1.0.0/ exists and is /1/'s target, and it has no copy.js. |
| 276 | const publishedOnlyEnv = "FIESTA_LINKS_PUBLISHED" |
| 277 | |
| 278 | // linkTree is the tree the link check reads: tree as site.Build made |
| 279 | // it, or, while the source is ahead of releases/ and publishedOnly is |
| 280 | // false, tree with dist served as the release to come (servingSource). |
| 281 | // It reports whether it fell back to dist. |
| 282 | func linkTree(t *testing.T, tree map[string][]byte, releases fs.FS, version string, dist map[string][]byte, publishedOnly bool) (map[string][]byte, bool) { |
| 283 | t.Helper() |
| 284 | if publishedOnly { |
| 285 | return tree, false |
| 286 | } |
| 287 | ahead, err := unpublished(releases, version, dist) |
| 288 | if err != nil { |
| 289 | t.Fatalf("reading releases/%s: %v", version, err) |
| 290 | } |
| 291 | if !ahead { |
| 292 | return tree, false |
| 293 | } |
| 294 | return servingSource(t, tree, version, dist), true |
| 295 | } |
| 296 | |
| 297 | // TestEveryLinkInTheSiteResolves: on every page of the built site, |
| 298 | // the landing page in each language, the design system's index, every |
| 299 | // example page and every frame, every href, src and srcset reaches a |
| 300 | // file the edge would serve, resolved against the page's own URL, |
| 301 | // through a directory index, the /1/ rewrite or a real file; every |
| 302 | // fragment names an id on the page it points at; and nothing links |
| 303 | // outside the site but the URLs listed above. The install text's URL |
| 304 | // is held to the same rule as the hreflang links: the site's own, |
| 305 | // through the rewrite to the newest release. |
| 306 | // |
| 307 | // While the source is unpublished (see unpublished), /1/ is resolved |
| 308 | // against the release the source builds, not the newest published one: |
| 309 | // the design system lists every file the source ships, linked through |
| 310 | // /1/, and the site goes out with the release that carries them. Read |
| 311 | // against 1.0.0 alone, every new file is a 404 from the day it is |
| 312 | // added to the day the release is cut. Once releases/ holds what the |
| 313 | // source builds, /1/ resolves against it as the edge will. With |
| 314 | // FIESTA_LINKS_PUBLISHED=1 there is no fallback: /1/ resolves against |
| 315 | // the newest published release only, as the edge serves the tree being |
| 316 | // shipped. That mode fails from a change to the source until the |
| 317 | // release that carries it is cut, by design, so it runs before a ship |
| 318 | // (make site-links-published) and not in the suite. |
| 319 | // |
| 320 | // The control comes first: a scratch site whose one page links a |
| 321 | // missing page, a missing fragment, a wrong path under the site's own |
| 322 | // origin and an outside URL that is not on the list. The check has to |
| 323 | // name all four, or a clean run of the real site proves nothing. A |
| 324 | // second control holds the unpublished stretch: a file the source |
| 325 | // builds resolves through /1/, and one it does not build still fails; |
| 326 | // and with the fallback off, the file only the source builds fails |
| 327 | // too. |
| 328 | func TestEveryLinkInTheSiteResolves(t *testing.T) { |
| 329 | var publishedOnly bool |
| 330 | switch v := os.Getenv(publishedOnlyEnv); v { |
| 331 | case "": |
| 332 | case "1": |
| 333 | publishedOnly = true |
| 334 | default: |
| 335 | t.Fatalf("%s=%q; set it to 1 or leave it unset", publishedOnlyEnv, v) |
| 336 | } |
| 337 | control := map[string][]byte{ |
| 338 | "_redirects": []byte("/1/* /1.0.0/:splat 200\n"), |
| 339 | "1.0.0/fiesta.css": []byte("body{}"), |
| 340 | "design-system/index.html": []byte(`<main id="main"><a href="../">up</a><link rel="stylesheet" href="/1/fiesta.css"></main>`), |
| 341 | "design-system/scratch.html": []byte(`<a href="/design-system/nope.html">missing</a>` + |
| 342 | `<a href="index.html#nowhere">no id</a>` + |
| 343 | `<a href="` + www.Origin + `/nope/">wrong path</a>` + |
| 344 | `<a href="https://example.com/">outside</a>` + |
| 345 | `<a href="#">bare</a><a href="/#main">main</a>`), |
| 346 | "index.html": []byte(`<main id="main"></main>`), |
| 347 | } |
| 348 | problems, _ := linkProblems(t, control) |
| 349 | for _, want := range []string{"/design-system/nope.html", "#nowhere", www.Origin + "/nope/", "https://example.com/"} { |
| 350 | found := false |
| 351 | for _, p := range problems { |
| 352 | found = found || strings.Contains(p, want) |
| 353 | } |
| 354 | if !found { |
| 355 | t.Errorf("the control's link to %s was not reported; the check would prove nothing. Reported:\n %s", want, strings.Join(problems, "\n ")) |
| 356 | } |
| 357 | } |
| 358 | if len(problems) != 4 { |
| 359 | t.Errorf("the control has 4 broken links and %d were reported:\n %s", len(problems), strings.Join(problems, "\n ")) |
| 360 | } |
| 361 | unpublishedControl(t) |
| 362 | if t.Failed() { |
| 363 | return |
| 364 | } |
| 365 | |
| 366 | dist, err := build.Dist(fiesta.FS(), fiesta.Version) |
| 367 | if err != nil { |
| 368 | t.Fatalf("building the release the source would publish: %v", err) |
| 369 | } |
| 370 | tree, fellBack := linkTree(t, builtSite(t), os.DirFS("../../releases"), fiesta.Version, dist, publishedOnly) |
| 371 | switch { |
| 372 | case publishedOnly: |
| 373 | t.Logf("%s=1: /1/ resolves against the newest published release only", publishedOnlyEnv) |
| 374 | case fellBack: |
| 375 | t.Logf("the source is ahead of releases/%s, so /1/ resolves against the release it builds", fiesta.Version) |
| 376 | } |
| 377 | problems, pages := linkProblems(t, tree) |
| 378 | for _, p := range problems { |
| 379 | t.Error(p) |
| 380 | } |
| 381 | read := map[string]bool{} |
| 382 | frames, examples := 0, 0 |
| 383 | for _, p := range pages { |
| 384 | read[p] = true |
| 385 | switch { |
| 386 | case strings.Contains(p, "/frames/"): |
| 387 | frames++ |
| 388 | case strings.HasPrefix(p, "design-system/") && p != "design-system/index.html": |
| 389 | examples++ |
| 390 | } |
| 391 | } |
| 392 | for _, l := range www.Languages { |
| 393 | if name := strings.TrimPrefix(pageURL(l), "/") + "index.html"; !read[name] { |
| 394 | t.Errorf("the landing page in %s, %s, was not read", l, name) |
| 395 | } |
| 396 | } |
| 397 | if !read["design-system/index.html"] { |
| 398 | t.Error("the design system's index was not read") |
| 399 | } |
| 400 | if examples == 0 || frames == 0 { |
| 401 | t.Errorf("%d example pages and %d frames were read; the design system has both", examples, frames) |
| 402 | } |
| 403 | } |
| 404 | |
| 405 | // unpublishedControl is the unpublished stretch on scratch trees. A |
| 406 | // page links /1/fiesta.css, which 1.0.0 has, /1/copy.js, which only |
| 407 | // the source builds, and /1/missing.js, which nothing does. Read |
| 408 | // against 1.0.0 alone the last two fail; served as the release to come |
| 409 | // only the last does. unpublished is held on both sides: a release |
| 410 | // equal to the source build (its stamp aside) is published, one with a |
| 411 | // file changed or none at all is not. |
| 412 | func unpublishedControl(t *testing.T) { |
| 413 | t.Helper() |
| 414 | dist := map[string][]byte{"fiesta.css": []byte("body{}"), "copy.js": []byte("0")} |
| 415 | dist["SHA256SUMS"] = sumsOf(dist) |
| 416 | tree := map[string][]byte{ |
| 417 | "_redirects": []byte("/1/* /1.0.0/:splat 200\n"), |
| 418 | "1.0.0/fiesta.css": []byte("body{}"), |
| 419 | "index.html": []byte(`<main id="main"><link rel="stylesheet" href="/1/fiesta.css">` + |
| 420 | `<script src="/1/copy.js"></script><script src="/1/missing.js"></script></main>`), |
| 421 | } |
| 422 | // releases/1.0.0/ as it is published: fiesta.css, and no copy.js. |
| 423 | published := map[string][]byte{"fiesta.css": []byte("body{}")} |
| 424 | published["SHA256SUMS"] = sumsOf(published) |
| 425 | releases := fstest.MapFS{} |
| 426 | for name, data := range published { |
| 427 | releases["1.0.0/"+name] = &fstest.MapFile{Data: data} |
| 428 | } |
| 429 | fallback, fellBack := linkTree(t, tree, releases, "1.0.0", dist, false) |
| 430 | if !fellBack { |
| 431 | t.Error("the unpublished control: a source build with copy.js, over a 1.0.0 without it, did not fall back to the release to come") |
| 432 | } |
| 433 | shipped, fellBack := linkTree(t, tree, releases, "1.0.0", dist, true) |
| 434 | if fellBack { |
| 435 | t.Errorf("the unpublished control: with %s=1 the check still fell back to the source build", publishedOnlyEnv) |
| 436 | } |
| 437 | for _, c := range []struct { |
| 438 | name string |
| 439 | tree map[string][]byte |
| 440 | want []string |
| 441 | }{ |
| 442 | {"against 1.0.0", tree, []string{"/1/copy.js", "/1/missing.js"}}, |
| 443 | {"as the release to come", fallback, []string{"/1/missing.js"}}, |
| 444 | {"as a new version's release", servingSource(t, tree, "1.1.0", dist), []string{"/1/missing.js"}}, |
| 445 | // The ship check: a file only the source builds is a 404 on the |
| 446 | // edge until it is released, and must fail here. |
| 447 | {"published only", shipped, []string{"/1/copy.js", "/1/missing.js"}}, |
| 448 | } { |
| 449 | problems, _ := linkProblems(t, c.tree) |
| 450 | var got []string |
| 451 | for _, p := range problems { |
| 452 | for _, link := range []string{"/1/fiesta.css", "/1/copy.js", "/1/missing.js"} { |
| 453 | if strings.Contains(p, "resolves to "+link+",") { |
| 454 | got = append(got, link) |
| 455 | } |
| 456 | } |
| 457 | } |
| 458 | if len(got) != len(problems) || !slices.Equal(got, c.want) { |
| 459 | t.Errorf("the unpublished control, %s, reported %q; want only %q broken", c.name, problems, c.want) |
| 460 | } |
| 461 | } |
| 462 | |
| 463 | stamp := &fstest.MapFile{Data: []byte("fiesta release\n")} |
| 464 | same := fstest.MapFS{"1.0.0/" + site.StampName: stamp} |
| 465 | changed := fstest.MapFS{"1.0.0/" + site.StampName: stamp} |
| 466 | for name, data := range dist { |
| 467 | same["1.0.0/"+name] = &fstest.MapFile{Data: data} |
| 468 | changed["1.0.0/"+name] = &fstest.MapFile{Data: data} |
| 469 | } |
| 470 | changed["1.0.0/copy.js"] = &fstest.MapFile{Data: []byte("1")} |
| 471 | changed["1.0.0/SHA256SUMS"] = &fstest.MapFile{Data: sumsOf(map[string][]byte{"fiesta.css": dist["fiesta.css"], "copy.js": []byte("1")})} |
| 472 | for _, c := range []struct { |
| 473 | name string |
| 474 | releases fs.FS |
| 475 | want bool |
| 476 | }{ |
| 477 | {"the same files", same, false}, |
| 478 | {"one file changed", changed, true}, |
| 479 | {"no release directory", fstest.MapFS{}, true}, |
| 480 | } { |
| 481 | got, err := unpublished(c.releases, "1.0.0", dist) |
| 482 | if err != nil || got != c.want { |
| 483 | t.Errorf("unpublished with %s = %v, %v; want %v", c.name, got, err, c.want) |
| 484 | } |
| 485 | } |
| 486 | } |
| 487 | |
amadan