| 1 | //go:build browser |
| 2 | |
| 3 | package gallery |
| 4 | |
| 5 | import ( |
| 6 | "context" |
| 7 | "fmt" |
| 8 | "net/http" |
| 9 | "sort" |
| 10 | "strings" |
| 11 | "testing" |
| 12 | "time" |
| 13 | |
| 14 | "github.com/chromedp/chromedp" |
| 15 | |
| 16 | "amadan.net/rastrillo/fiesta" |
| 17 | "amadan.net/rastrillo/fiesta/internal/axe" |
| 18 | "amadan.net/rastrillo/fiesta/internal/harness" |
| 19 | ) |
| 20 | |
| 21 | // The gallery's own pages under axe. a11y_browser_test.go scans every |
| 22 | // state's frame document, which is where the examples' markup lives; |
| 23 | // this file scans the pages around them, which carry the design |
| 24 | // system's explanations: the families' headings and sentences, the |
| 25 | // rules at the head of the primitives, and the notes under the states. |
| 26 | |
| 27 | // a11yPages is what the gallery scan reads in every theme, scheme and |
| 28 | // locale it covers: one page per page kind, read off pageKinds() so a |
| 29 | // kind added later is scanned the day it lands (the families' pages, |
| 30 | // form.html among them, included), then the pages no kind covers. |
| 31 | // button carries notes on its states and the widget, dropdown the |
| 32 | // idiom's note on one state, signin the summary, a note per screen and |
| 33 | // the warning callout, format-money a format's blurb, layout-sidebar a |
| 34 | // shell framed at the page width with its open button, modal the |
| 35 | // primitive with its live link; then two documents a reader opens from |
| 36 | // the Demos, the demo application's index and the modal route. |
| 37 | func a11yPages(s *site) []string { |
| 38 | var out []string |
| 39 | for _, pk := range s.pageKinds() { |
| 40 | out = append(out, pk.File) |
| 41 | } |
| 42 | return append(out, "button.html", "dropdown.html", "signin.html", "format-money.html", "layout-sidebar.html", "modal.html", "demo.html", "demo-modal.html") |
| 43 | } |
| 44 | |
| 45 | // TestA11yScansTheGalleryDay, Plain and Signal are the gate over the gallery's own pages, |
| 46 | // which carry the design system's explanations around the samples: |
| 47 | // a11yPages in day, plain and signal, with data-theme light and dark, |
| 48 | // in English and in Arabic, at 1280 (where the rail, the bar and the |
| 49 | // Overview's content are all on the page), with the WCAG 2.2 AA tag |
| 50 | // set. Expected: no violations. The rules axe could not decide are |
| 51 | // logged per leg, never silently dropped. It extends Fiesta's |
| 52 | // TestTheGalleryPagesPassAxe, which scanned seven pages in English and |
| 53 | // the Arabic index, to every page kind and the Demos. The frames' |
| 54 | // documents are TestEveryExamplePassesAxe's. |
| 55 | // |
| 56 | // Preservation of the pages scan, already passing when written. The |
| 57 | // named mutation is .ds-lead's colour set to --rst-line in |
| 58 | // gallery.css, which axe reports as color-contrast on every page with a |
| 59 | // lead. The rail's accessible name removed is not one: two unnamed |
| 60 | // navigation landmarks are axe's landmark-unique, a best-practice rule |
| 61 | // outside the WCAG 2.2 AA tag set this gate runs. |
| 62 | // |
| 63 | // It is one test per theme, so the suite's slices can run them on |
| 64 | // shards of their own. Each checks that the themes are exactly the |
| 65 | // three the tests name, so a theme added later cannot go unscanned, |
| 66 | // and counts its own scans, as the one test counted all of them. |
| 67 | func TestA11yScansTheGalleryDay(t *testing.T) { a11yScansTheGallery(t, "day") } |
| 68 | func TestA11yScansTheGalleryPlain(t *testing.T) { a11yScansTheGallery(t, "plain") } |
| 69 | func TestA11yScansTheGallerySignal(t *testing.T) { a11yScansTheGallery(t, "signal") } |
| 70 | |
| 71 | // a11yScanThemes are the themes the three tests above scan, one each. |
| 72 | var a11yScanThemes = []string{"day", "plain", "signal"} |
| 73 | |
| 74 | // a11yScansTheGallery is the gallery's pages scanned in one theme. |
| 75 | func a11yScansTheGallery(t *testing.T, only string) { |
| 76 | if got := fiesta.ThemeNames(); strings.Join(got, " ") != strings.Join(a11yScanThemes, " ") { |
| 77 | t.Fatalf("the themes are %v and the gallery's pages are scanned in %v, one test each; a theme needs a test of its own", got, a11yScanThemes) |
| 78 | } |
| 79 | rig := harness.New(t, func(string) http.Handler { return treeHandler(t) }) |
| 80 | ctx, cancel := context.WithTimeout(rig.Context(), 18*time.Minute) |
| 81 | defer cancel() |
| 82 | pages := a11yPages(newTestSite(t, fiesta.FS(), mountPath)) |
| 83 | |
| 84 | scans, failed := 0, 0 |
| 85 | for _, theme := range []string{only} { |
| 86 | for _, locale := range []string{"en", "ar"} { |
| 87 | name := theme + " " + locale |
| 88 | incomplete := map[string]int{} |
| 89 | start := time.Now() |
| 90 | for _, file := range pages { |
| 91 | dir := theme + "/" + locale + "/" |
| 92 | where := name + " " + file |
| 93 | mustRun(t, ctx, |
| 94 | chromedp.EmulateViewport(1280, 900), |
| 95 | chromedp.Navigate(rig.Origin+href(dir+file)), |
| 96 | chromedp.WaitReady("body"), |
| 97 | ) |
| 98 | // The body's ground is read back after each flip, and |
| 99 | // the two must differ: a dark scan of a page still |
| 100 | // painted light would be the light scan twice. |
| 101 | ground := map[string]string{} |
| 102 | for _, scheme := range []string{"light", "dark"} { |
| 103 | var bg string |
| 104 | mustRun(t, ctx, chromedp.Evaluate(fmt.Sprintf(`document.documentElement.setAttribute("data-theme", %q); getComputedStyle(document.body).backgroundColor`, scheme), &bg)) |
| 105 | ground[scheme] = bg |
| 106 | res, err := axe.Scan(ctx, "", axe.DefaultTags()) |
| 107 | if err != nil { |
| 108 | t.Fatalf("%s %s: %v", where, scheme, err) |
| 109 | } |
| 110 | scans++ |
| 111 | for _, id := range res.Incomplete { |
| 112 | incomplete[id]++ |
| 113 | } |
| 114 | failed += report(t, where+" "+scheme, res.Violations) |
| 115 | } |
| 116 | if ground["light"] == ground["dark"] { |
| 117 | t.Fatalf("%s: the body paints %s under both schemes, so the flip to dark did not take", where, ground["light"]) |
| 118 | } |
| 119 | } |
| 120 | var ids []string |
| 121 | for id, n := range incomplete { |
| 122 | ids = append(ids, fmt.Sprintf("%s x%d", id, n)) |
| 123 | } |
| 124 | sort.Strings(ids) |
| 125 | t.Logf("%s: %d pages scanned in %s; needs review, axe could not decide: %v", name, len(pages), time.Since(start).Round(time.Second), ids) |
| 126 | } |
| 127 | } |
| 128 | if want := 2 * len(pages) * 2; scans != want { |
| 129 | t.Fatalf("%d scans, want %d", scans, want) |
| 130 | } |
| 131 | t.Logf("%d scans, %d violations", scans, failed) |
| 132 | rig.Screen("main", "the last page") |
| 133 | } |
| 134 | |
| 135 | // shellDemo is one layout example's document in the tree, in theme and |
| 136 | // English, at the state the manifest opens it on: a shell demo. |
| 137 | func shellDemo(t *testing.T, theme, name string) string { |
| 138 | t.Helper() |
| 139 | for _, e := range manifest(t).Examples { |
| 140 | if e.Name == name { |
| 141 | if !e.Document { |
| 142 | t.Fatalf("%s is not a layout document; it has no shell demo", name) |
| 143 | } |
| 144 | return frameFile(theme, "en", name, e.Default) |
| 145 | } |
| 146 | } |
| 147 | t.Fatalf("the manifest has no %s", name) |
| 148 | return "" |
| 149 | } |
| 150 | |
| 151 | // shownJS reports whether the first element matching a selector has a |
| 152 | // box on screen. The patency check and its control both read it, so |
| 153 | // the two are the same instrument pointed at two moments. It asks |
| 154 | // checkVisibility as well as for a box: the content of a closed |
| 155 | // details still lays out when asked (the engine hides it with |
| 156 | // content-visibility), so its box alone reads as shown. |
| 157 | const shownJS = `((sel) => { |
| 158 | const el = document.querySelector(sel); |
| 159 | if (!el) return false; |
| 160 | const r = el.getBoundingClientRect(); |
| 161 | return el.checkVisibility() && r.width > 0 && r.height > 0; |
| 162 | })(%q)` |
| 163 | |
| 164 | // settleShellJS waits out the document's finite animations and any |
| 165 | // view transition. The cards fade in and a narrow navigation between |
| 166 | // two shell documents slides; axe reading either mid-flight measures |
| 167 | // text at partial opacity, blended colours no reader sees once it has |
| 168 | // landed. Finite animations only: a spinner's never finishes. |
| 169 | const settleShellJS = `(async () => { |
| 170 | if (document.activeViewTransition) await document.activeViewTransition.finished.catch(() => null); |
| 171 | await Promise.all(document.getAnimations() |
| 172 | .filter(a => a.effect && a.effect.getComputedTiming().endTime !== Infinity) |
| 173 | .map(a => a.finished.catch(() => null))); |
| 174 | return true; |
| 175 | })()` |
| 176 | |
| 177 | // targetSizeJS sets whether the engine on the page runs axe's |
| 178 | // target-size rule. The engine's own run is kept aside the first time, |
| 179 | // so a scan with the rule is always the engine as shipped. |
| 180 | const targetSizeJS = `((on) => { |
| 181 | if (!window.__axeRun) window.__axeRun = window.axe.run.bind(window.axe); |
| 182 | window.axe.run = on ? window.__axeRun : (target, opts) => |
| 183 | window.__axeRun(target, Object.assign({}, opts, {rules: Object.assign({}, opts && opts.rules, {"target-size": {enabled: false}})})); |
| 184 | return true; |
| 185 | })(%t)` |
| 186 | |
| 187 | // TestA11yScansTheShellsCollapsed is the scan the others do not cover: |
| 188 | // every shell demo below its 800px breakpoint. |
| 189 | // |
| 190 | // The other scans run the shell documents at 1280, where the topbar's |
| 191 | // collapse control does not exist at all and the sidebar and console |
| 192 | // show their rails beside the page. So the views a reader on a phone |
| 193 | // meets first are views axe would never see. The topbar is scanned |
| 194 | // with its card open. The sidebar and console are two views of every |
| 195 | // URL below 800px, so both of their documents are scanned, in every |
| 196 | // theme and scheme: the index (the rail is the page), with the |
| 197 | // console's card and the sidebar's profile card open as well, and the |
| 198 | // content page (the back control and the page, no rail). Then the |
| 199 | // sidebar's profile card at the rail's foot on a desktop, open with |
| 200 | // the language menu inside it, in every theme and scheme. |
| 201 | // |
| 202 | // It also measures what axe will not: target size. The topbar's |
| 203 | // summary against WCAG 2.2's 24x24 floor, and the back control against |
| 204 | // the 44px the touch block promises (390px is inside its width half). |
| 205 | // |
| 206 | // Ported from Rastrillo onto Fiesta's layout documents: layout-topbar, |
| 207 | // layout-sidebar and layout-console are the index documents and the |
| 208 | // two -page examples the content pages. A scheme is painted as |
| 209 | // Fiesta's other scans paint it, by data-theme on the root, and each |
| 210 | // paint is checked to have changed the ground. Every disclosure is |
| 211 | // checked shut before the click and open after it, so a summary that |
| 212 | // stopped opening fails rather than being scanned shut and reported |
| 213 | // clean. |
| 214 | // |
| 215 | // A preservation port, already passing when written: Fiesta's shells |
| 216 | // were clean collapsed. Its two mutations, each one rule appended to |
| 217 | // frameStyle and reverted: [rst-shell-back] a { color: #bbb } is |
| 218 | // color-contrast on every content page in light, which no other scan |
| 219 | // sees, the back control being hidden above 800px; and the back |
| 220 | // control held to 30px tall fails the 44px check on every content |
| 221 | // page. |
| 222 | func TestA11yScansTheShellsCollapsed(t *testing.T) { |
| 223 | rig := harness.New(t, func(string) http.Handler { return treeHandler(t) }) |
| 224 | ctx, cancel := context.WithTimeout(rig.Context(), 15*time.Minute) |
| 225 | defer cancel() |
| 226 | |
| 227 | total, scans := 0, 0 |
| 228 | incomplete := map[string]int{} |
| 229 | |
| 230 | shown := func(where, sel string) bool { |
| 231 | t.Helper() |
| 232 | var ok bool |
| 233 | if err := chromedp.Run(ctx, chromedp.Evaluate(fmt.Sprintf(shownJS, sel), &ok)); err != nil { |
| 234 | t.Fatalf("%s: reading whether %q is on screen: %v", where, sel, err) |
| 235 | } |
| 236 | return ok |
| 237 | } |
| 238 | load := func(where, file string, w, h int64, ready string) { |
| 239 | t.Helper() |
| 240 | if err := chromedp.Run(ctx, |
| 241 | chromedp.EmulateViewport(w, h), |
| 242 | chromedp.Navigate(rig.Origin+href(file)), |
| 243 | chromedp.WaitVisible(ready, chromedp.ByQuery), |
| 244 | ); err != nil { |
| 245 | t.Fatalf("%s: loading: %v", where, err) |
| 246 | } |
| 247 | } |
| 248 | // paint puts the document in scheme, and proves the flip took: the |
| 249 | // body's ground under the other scheme differs from its ground |
| 250 | // under this one, or a dark scan would be the light scan twice. |
| 251 | paint := func(where, scheme string) { |
| 252 | t.Helper() |
| 253 | other := map[string]string{"light": "dark", "dark": "light"}[scheme] |
| 254 | var away, here string |
| 255 | const flip = `document.documentElement.setAttribute("data-theme", %q); getComputedStyle(document.body).backgroundColor` |
| 256 | mustRun(t, ctx, |
| 257 | chromedp.Evaluate(fmt.Sprintf(flip, other), &away), |
| 258 | chromedp.Evaluate(fmt.Sprintf(flip, scheme), &here)) |
| 259 | if away == here { |
| 260 | t.Fatalf("%s: the body paints %s under both schemes, so the flip to %s did not take", where, here, scheme) |
| 261 | } |
| 262 | } |
| 263 | // scan runs axe over the whole document in scheme, with or without |
| 264 | // the target-size rule. |
| 265 | scan := func(where, scheme string, targetSize bool) { |
| 266 | t.Helper() |
| 267 | var loaded bool |
| 268 | mustRun(t, ctx, chromedp.Evaluate(settleShellJS, nil, awaitPromise), |
| 269 | chromedp.Evaluate(`typeof window.axe !== "undefined"`, &loaded)) |
| 270 | if !loaded { |
| 271 | mustRun(t, ctx, chromedp.Evaluate(axe.Script(), nil)) |
| 272 | } |
| 273 | mustRun(t, ctx, chromedp.Evaluate(fmt.Sprintf(targetSizeJS, targetSize), nil)) |
| 274 | paint(where, scheme) |
| 275 | res, err := axe.Scan(ctx, "", axe.DefaultTags()) |
| 276 | if err != nil { |
| 277 | t.Fatalf("%s: %v", where, err) |
| 278 | } |
| 279 | scans++ |
| 280 | for _, id := range res.Incomplete { |
| 281 | incomplete[id]++ |
| 282 | } |
| 283 | total += report(t, where, res.Violations) |
| 284 | } |
| 285 | // box is a control's size and visible label. |
| 286 | box := func(sel string) (w, h int, text string) { |
| 287 | t.Helper() |
| 288 | var got struct { |
| 289 | W, H int |
| 290 | Text string |
| 291 | } |
| 292 | at(t, ctx, fmt.Sprintf(`((sel) => { |
| 293 | const s = document.querySelector(sel); |
| 294 | const r = s.getBoundingClientRect(); |
| 295 | return JSON.stringify({W: Math.round(r.width), H: Math.round(r.height), Text: s.innerText.trim()}); |
| 296 | })(%q)`, sel), &got) |
| 297 | return got.W, got.H, got.Text |
| 298 | } |
| 299 | // open clicks a disclosure and asserts what it reveals, with the |
| 300 | // control first: the same reading one click earlier, where the |
| 301 | // answer is known. A patency check hard-wired true passes after the |
| 302 | // click and fails here. |
| 303 | open := func(where, summary, revealed string) { |
| 304 | t.Helper() |
| 305 | if shown(where, revealed) { |
| 306 | t.Errorf("%s: %q is on screen with the disclosure still shut, so it is not behind the disclosure and its being shown proves nothing about the click", where, revealed) |
| 307 | } |
| 308 | if err := chromedp.Run(ctx, chromedp.Click(summary, chromedp.ByQuery)); err != nil { |
| 309 | t.Fatalf("%s: opening: %v", where, err) |
| 310 | } |
| 311 | if !shown(where, revealed) { |
| 312 | t.Fatalf("%s: clicking the disclosure did not bring %q on screen; axe would scan the collapsed document as the disclosed one", where, revealed) |
| 313 | } |
| 314 | } |
| 315 | schemes := []string{"light", "dark"} |
| 316 | themes := fiesta.ThemeNames() |
| 317 | |
| 318 | // The topbar: its card, open. |
| 319 | for _, scheme := range schemes { |
| 320 | where := themes[0] + "/en topbar shell at 390px, card open (" + scheme + ")" |
| 321 | load(where, shellDemo(t, themes[0], "layout-topbar"), 390, 780, "[rst-shell-menu] > summary") |
| 322 | open(where, "[rst-shell-menu] > summary", "[rst-shell-tail] [rst-shell-nav] a") |
| 323 | scan(where, scheme, true) |
| 324 | w, h, text := box("[rst-shell-menu] > summary") |
| 325 | if w < 24 || h < 24 { |
| 326 | t.Errorf("%s: the disclosure's target is %dx%dpx, under WCAG 2.2 SC 2.5.8's 24x24", where, w, h) |
| 327 | } |
| 328 | if text == "" { |
| 329 | t.Errorf("%s: the disclosure has no visible label; the icon beside it is aria-hidden, so the control would have no accessible name at all", where) |
| 330 | } |
| 331 | } |
| 332 | |
| 333 | // The sidebar and console: both documents, every theme and scheme. |
| 334 | for _, shell := range []string{"sidebar", "console"} { |
| 335 | for _, theme := range themes { |
| 336 | for _, scheme := range schemes { |
| 337 | where := theme + "/en " + shell + " index at 390px (" + scheme + ")" |
| 338 | load(where, shellDemo(t, theme, "layout-"+shell), 390, 780, "[rst-shell-rail]") |
| 339 | // Patency: the document is the index before axe reads it. |
| 340 | if rail, main := shown(where, "[rst-shell-rail] [rst-shell-nav] a"), shown(where, "[rst-shell-main]"); !rail || main { |
| 341 | t.Fatalf("%s: not the index (rail shown %v, main shown %v); axe would scan the wrong view", where, rail, main) |
| 342 | } |
| 343 | scan(where, scheme, true) |
| 344 | // The whole document with the card open, every rule but |
| 345 | // target-size. With the card open everything outside it |
| 346 | // lies under the summary's light-dismiss layer, which axe |
| 347 | // cannot see (it works out what covers a target from |
| 348 | // element boxes): a rail row half under the card reads as |
| 349 | // a target cut short when no tap can reach it at all. The |
| 350 | // rows' size is measured with the card shut, just above. |
| 351 | if shell == "console" { |
| 352 | where := theme + "/en console index at 390px, card open (" + scheme + ")" |
| 353 | // On the index the account's links are laid out flat |
| 354 | // in the card, so a link is what the click reveals. |
| 355 | open(where, "[rst-shell-menu] > summary", "[rst-shell-tail] [rst-shell-account] [rst-dropdown-menu] a") |
| 356 | scan(where, scheme, false) |
| 357 | } else { |
| 358 | where := theme + "/en sidebar index at 390px, profile card open (" + scheme + ")" |
| 359 | open(where, "[rst-shell-profile] > summary", "[rst-shell-profile] [rst-shell-who] [rst-person-name]") |
| 360 | open(where+", language open", "[rst-shell-profile] [rst-locale] > summary", "[rst-shell-profile] [rst-locale] [rst-dropdown-menu] a") |
| 361 | scan(where, scheme, false) |
| 362 | } |
| 363 | |
| 364 | where = theme + "/en " + shell + " content page at 390px (" + scheme + ")" |
| 365 | load(where, shellDemo(t, theme, "layout-"+shell+"-page"), 390, 780, "[rst-shell-back] a") |
| 366 | if shown(where, "[rst-shell-rail] [rst-shell-nav] a") { |
| 367 | t.Fatalf("%s: the rail's nav is on screen on a content page; axe would scan the index", where) |
| 368 | } |
| 369 | scan(where, scheme, true) |
| 370 | w, h, text := box("[rst-shell-back] a") |
| 371 | if w < 44 || h < 44 { |
| 372 | t.Errorf("%s: the back control is %dx%dpx; the touch block promises 44x44", where, w, h) |
| 373 | } |
| 374 | if text == "" { |
| 375 | t.Errorf("%s: the back control has no visible label", where) |
| 376 | } |
| 377 | } |
| 378 | } |
| 379 | } |
| 380 | |
| 381 | // The sidebar's profile menu on a desktop, the drop-up at the rail's |
| 382 | // foot, open with the language inside it, in every theme and scheme. |
| 383 | // target-size is off for the same layer as above: the page under |
| 384 | // the card is behind it. |
| 385 | for _, theme := range themes { |
| 386 | for _, scheme := range schemes { |
| 387 | where := theme + "/en sidebar content page at 1280px, profile card open (" + scheme + ")" |
| 388 | load(where, shellDemo(t, theme, "layout-sidebar-page"), 1280, 720, "[rst-shell-profile] > summary") |
| 389 | open(where, "[rst-shell-profile] > summary", "[rst-shell-profile] [rst-shell-who] [rst-person-name]") |
| 390 | open(where+", language open", "[rst-shell-profile] [rst-locale] > summary", "[rst-shell-profile] [rst-locale] [rst-dropdown-menu] a") |
| 391 | scan(where, scheme, false) |
| 392 | } |
| 393 | } |
| 394 | |
| 395 | if want := 2 + 2*len(themes)*2*3 + len(themes)*2; scans != want { |
| 396 | t.Errorf("%d scans; want %d", scans, want) |
| 397 | } |
| 398 | var ids []string |
| 399 | for id, n := range incomplete { |
| 400 | ids = append(ids, fmt.Sprintf("%s x%d", id, n)) |
| 401 | } |
| 402 | sort.Strings(ids) |
| 403 | t.Logf("%d scans at 390px and the desktop profile card, %d violations; needs review, axe could not decide: %v", scans, total, ids) |
| 404 | } |
| 405 | |
amadan