| 1 | package gallery |
| 2 | |
| 3 | import ( |
| 4 | "encoding/json" |
| 5 | "html/template" |
| 6 | "io/fs" |
| 7 | "regexp" |
| 8 | "slices" |
| 9 | "sort" |
| 10 | "strings" |
| 11 | "sync" |
| 12 | "testing" |
| 13 | |
| 14 | xhtml "golang.org/x/net/html" |
| 15 | |
| 16 | "amadan.net/rastrillo/fiesta" |
| 17 | ) |
| 18 | |
| 19 | // The translation gates: every id a render asks for is translated in |
| 20 | // all eleven catalogs, every chrome string is one a render asks for, |
| 21 | // no English the gallery says in its own voice reaches a translated |
| 22 | // page, and the page templates write no English of their own. |
| 23 | |
| 24 | // The two classes of document in a theme and locale directory. A page |
| 25 | // is a page kind's file or an example's page: the gallery's chrome and |
| 26 | // prose, in the page's language through the catalogs. A sample document |
| 27 | // is a frame or one of the extra documents (the demo application's |
| 28 | // four and the modal route), whose words are the examples' own, |
| 29 | // translated through the sample catalogs. The leak sweep reads both. |
| 30 | const ( |
| 31 | docPage = "page" |
| 32 | docSample = "sample document" |
| 33 | ) |
| 34 | |
| 35 | // documentKind classes one path of the tree: docPage, docSample, or "" |
| 36 | // for anything else (the assets, the tree root). |
| 37 | func (s *site) documentKind(name string) string { |
| 38 | parts := strings.SplitN(name, "/", 3) |
| 39 | if len(parts) != 3 || !slices.Contains(fiesta.ThemeNames(), parts[0]) || !known(parts[1]) { |
| 40 | return "" |
| 41 | } |
| 42 | rest := parts[2] |
| 43 | if strings.HasPrefix(rest, "frames/") || slices.Contains(s.extraDocumentNames(), rest) { |
| 44 | return docSample |
| 45 | } |
| 46 | for _, pk := range s.pageKinds() { |
| 47 | if pk.File == rest { |
| 48 | return docPage |
| 49 | } |
| 50 | } |
| 51 | if stem, ok := strings.CutSuffix(rest, ".html"); ok { |
| 52 | if _, ok := s.m.Entry(stem); ok { |
| 53 | return docPage |
| 54 | } |
| 55 | } |
| 56 | return "" |
| 57 | } |
| 58 | |
| 59 | // tracedIDs is every id one full render asks for. One render serves |
| 60 | // every gate that reads it. |
| 61 | func tracedIDs(t *testing.T) map[string]bool { |
| 62 | t.Helper() |
| 63 | tracedOnce.Do(func() { |
| 64 | seen := map[string]bool{} |
| 65 | var mu sync.Mutex |
| 66 | stop := setTrace(func(id string) { |
| 67 | mu.Lock() |
| 68 | seen[id] = true |
| 69 | mu.Unlock() |
| 70 | }) |
| 71 | _, tracedErr = Render(fiesta.FS(), Options{}) |
| 72 | stop() |
| 73 | tracedSet = seen |
| 74 | }) |
| 75 | if tracedErr != nil { |
| 76 | t.Fatalf("Render: %v", tracedErr) |
| 77 | } |
| 78 | return tracedSet |
| 79 | } |
| 80 | |
| 81 | var ( |
| 82 | tracedOnce sync.Once |
| 83 | tracedSet map[string]bool |
| 84 | tracedErr error |
| 85 | ) |
| 86 | |
| 87 | // englishStrings is EnglishStrings over the source tree, failed loudly. |
| 88 | func englishStrings(t *testing.T) map[string]string { |
| 89 | t.Helper() |
| 90 | m, err := EnglishStrings(fiesta.FS()) |
| 91 | if err != nil { |
| 92 | t.Fatal(err) |
| 93 | } |
| 94 | return m |
| 95 | } |
| 96 | |
| 97 | // loadedCatalogs is the eleven catalogs, failed loudly. |
| 98 | func loadedCatalogs(t *testing.T) map[string]catalog { |
| 99 | t.Helper() |
| 100 | cs, err := catalogs() |
| 101 | if err != nil { |
| 102 | t.Fatal(err) |
| 103 | } |
| 104 | return cs |
| 105 | } |
| 106 | |
| 107 | // nonEnglish is every locale a catalog answers for. |
| 108 | func nonEnglish() []string { |
| 109 | var out []string |
| 110 | for _, l := range Locales { |
| 111 | if l != "en" { |
| 112 | out = append(out, l) |
| 113 | } |
| 114 | } |
| 115 | return out |
| 116 | } |
| 117 | |
| 118 | // placeholderName is one {name} in a string. |
| 119 | var placeholderName = regexp.MustCompile(`\{[a-z]+\}`) |
| 120 | |
| 121 | // TestEveryProseKeyIsTranslated: every id a full render asks for, the |
| 122 | // chrome's, the prose's and the labels', has a non-empty translation in |
| 123 | // all eleven catalogs, with every {placeholder} of its English kept and |
| 124 | // none added; no catalog carries an id no render asks for; the catalog |
| 125 | // directory holds the eleven and drafts.json, with no en catalog and |
| 126 | // no locale the gallery does not render; and drafts.json names only |
| 127 | // entries the catalogs carry, each drafted from the English the id has |
| 128 | // now. |
| 129 | func TestEveryProseKeyIsTranslated(t *testing.T) { |
| 130 | ids := tracedIDs(t) |
| 131 | english := englishStrings(t) |
| 132 | cs := loadedCatalogs(t) |
| 133 | sorted := make([]string, 0, len(ids)) |
| 134 | for id := range ids { |
| 135 | sorted = append(sorted, id) |
| 136 | } |
| 137 | sort.Strings(sorted) |
| 138 | for _, id := range sorted { |
| 139 | if strings.HasPrefix(id, "samples.") { |
| 140 | // A sample string's translation is in the sample catalogs, |
| 141 | // which TestEverySampleStringIsTranslated holds. |
| 142 | continue |
| 143 | } |
| 144 | en, ok := english[id] |
| 145 | if !ok { |
| 146 | t.Errorf("a render asked for %s, which EnglishStrings does not list, so the exporter never sees it", id) |
| 147 | continue |
| 148 | } |
| 149 | want := placeholderName.FindAllString(en, -1) |
| 150 | sort.Strings(want) |
| 151 | for _, l := range nonEnglish() { |
| 152 | tr := cs[l][id] |
| 153 | if strings.TrimSpace(tr) == "" { |
| 154 | t.Errorf("i18n/%s.json has no translation of %s (%q)", l, id, en) |
| 155 | continue |
| 156 | } |
| 157 | got := placeholderName.FindAllString(tr, -1) |
| 158 | sort.Strings(got) |
| 159 | if !slices.Equal(got, want) { |
| 160 | t.Errorf("i18n/%s.json: %s carries the placeholders %v, its English %v", l, id, got, want) |
| 161 | } |
| 162 | } |
| 163 | } |
| 164 | for _, l := range nonEnglish() { |
| 165 | for id := range cs[l] { |
| 166 | if !ids[id] { |
| 167 | t.Errorf("i18n/%s.json carries %s, which no render asks for", l, id) |
| 168 | } |
| 169 | } |
| 170 | } |
| 171 | |
| 172 | entries, err := fs.ReadDir(catalogFiles, "i18n") |
| 173 | if err != nil { |
| 174 | t.Fatal(err) |
| 175 | } |
| 176 | for _, e := range entries { |
| 177 | name := e.Name() |
| 178 | locale, _ := strings.CutSuffix(name, ".json") |
| 179 | switch { |
| 180 | case name == "drafts.json" || name == "money.json": |
| 181 | case strings.HasPrefix(locale, "samples."): |
| 182 | if l := strings.TrimPrefix(locale, "samples."); l == "en" || !known(l) { |
| 183 | t.Errorf("i18n/ carries %s, which is no translated locale's sample catalog", name) |
| 184 | } |
| 185 | case locale == "en": |
| 186 | t.Errorf("i18n/ carries %s; English is copy.go, the prose and the manifest", name) |
| 187 | case !known(locale): |
| 188 | t.Errorf("i18n/ carries %s, which is no locale the gallery renders", name) |
| 189 | } |
| 190 | } |
| 191 | |
| 192 | b, err := fs.ReadFile(catalogFiles, "i18n/drafts.json") |
| 193 | if err != nil { |
| 194 | t.Fatal(err) |
| 195 | } |
| 196 | var drafts map[string]struct { |
| 197 | English string `json:"english"` |
| 198 | Locales []string `json:"locales"` |
| 199 | } |
| 200 | if err := json.Unmarshal(b, &drafts); err != nil { |
| 201 | t.Fatalf("i18n/drafts.json: %v", err) |
| 202 | } |
| 203 | for id, d := range drafts { |
| 204 | if strings.HasPrefix(id, "samples.") { |
| 205 | continue |
| 206 | } |
| 207 | if d.English != english[id] { |
| 208 | t.Errorf("i18n/drafts.json: %s was drafted from %q, and its English is now %q", id, d.English, english[id]) |
| 209 | } |
| 210 | for _, l := range d.Locales { |
| 211 | if l == "en" || !known(l) || cs[l][id] == "" { |
| 212 | t.Errorf("i18n/drafts.json names a %s draft of %s that no catalog carries", l, id) |
| 213 | } |
| 214 | } |
| 215 | } |
| 216 | t.Logf("%d ids in %d catalogs, %d of them drafted somewhere", len(sorted), len(cs), len(drafts)) |
| 217 | } |
| 218 | |
| 219 | // TestEveryCopyKeyIsRendered: every id copy.go files is one a full |
| 220 | // render asks for, so a key a rewrite left behind is pruned rather |
| 221 | // than translated eleven times over. |
| 222 | func TestEveryCopyKeyIsRendered(t *testing.T) { |
| 223 | ids := tracedIDs(t) |
| 224 | keys := make([]string, 0, len(text)) |
| 225 | for id := range text { |
| 226 | keys = append(keys, id) |
| 227 | } |
| 228 | sort.Strings(keys) |
| 229 | for _, id := range keys { |
| 230 | if !ids[id] { |
| 231 | t.Errorf("copy.go files %s (%q), which no render asks for; prune it", id, text[id]) |
| 232 | } |
| 233 | } |
| 234 | } |
| 235 | |
| 236 | // leakFloor is the length, in runes, from which a string is a sentence |
| 237 | // and is looked for anywhere inside a run of text, as Rastrillo's |
| 238 | // sweep has it: a sentence does not occur inside other words by |
| 239 | // chance. Every traced string, sentence or word, is also looked for as |
| 240 | // a whole run of text or a whole spoken attribute value, which is how |
| 241 | // a short word of the chrome ("Copy", "Overview") is caught without |
| 242 | // matching inside the words around it. A string a locale rightly keeps |
| 243 | // as it is, is in leakUnchanged. |
| 244 | const leakFloor = 12 |
| 245 | |
| 246 | // leakSentences is the English the sweep looks for inside runs: each |
| 247 | // traced string's runs between placeholders, of leakFloor runes or |
| 248 | // more, whitespace collapsed, bar leakExempt. |
| 249 | func leakSentences(ids map[string]bool, english map[string]string) []string { |
| 250 | seen := map[string]bool{} |
| 251 | var out []string |
| 252 | for id := range ids { |
| 253 | en := english[id] |
| 254 | if leakExempt(en) { |
| 255 | continue |
| 256 | } |
| 257 | for _, run := range placeholderName.Split(en, -1) { |
| 258 | run = squash(run) |
| 259 | if len([]rune(run)) >= leakFloor && !leakExempt(run) && !seen[run] { |
| 260 | seen[run] = true |
| 261 | out = append(out, run) |
| 262 | } |
| 263 | } |
| 264 | } |
| 265 | sort.Strings(out) |
| 266 | return out |
| 267 | } |
| 268 | |
| 269 | // leakSweep is everything the sweep looks for: the sentences, inside |
| 270 | // any run; every traced string without placeholders, as a whole run; |
| 271 | // and every traced string with them, as a whole run with the |
| 272 | // placeholders filled by anything. |
| 273 | type leakSweep struct { |
| 274 | sentences []string |
| 275 | whole map[string]bool |
| 276 | patterns []leakPattern |
| 277 | } |
| 278 | |
| 279 | // leakPattern is one string with placeholders: its English, the longest |
| 280 | // run of it between placeholders (a cheap test before the expression), |
| 281 | // and the expression a whole run matches when it is that string filled. |
| 282 | type leakPattern struct { |
| 283 | en, key string |
| 284 | re *regexp.Regexp |
| 285 | } |
| 286 | |
| 287 | // newLeakSweep builds the sweep from the traced ids. A string whose |
| 288 | // only text between placeholders is punctuation ("{example}: {state}") |
| 289 | // says no English of its own and is not looked for. |
| 290 | func newLeakSweep(ids map[string]bool, english map[string]string) leakSweep { |
| 291 | w := leakSweep{sentences: leakSentences(ids, english), whole: map[string]bool{}} |
| 292 | for id := range ids { |
| 293 | en := squash(english[id]) |
| 294 | if en == "" || leakExempt(en) { |
| 295 | continue |
| 296 | } |
| 297 | if !placeholderName.MatchString(en) { |
| 298 | w.whole[en] = true |
| 299 | continue |
| 300 | } |
| 301 | parts := placeholderName.Split(en, -1) |
| 302 | if !templateLetter.MatchString(strings.Join(parts, "")) { |
| 303 | continue |
| 304 | } |
| 305 | quoted := make([]string, len(parts)) |
| 306 | key := "" |
| 307 | for i, part := range parts { |
| 308 | quoted[i] = regexp.QuoteMeta(part) |
| 309 | if len(strings.TrimSpace(part)) > len(key) { |
| 310 | key = strings.TrimSpace(part) |
| 311 | } |
| 312 | } |
| 313 | w.patterns = append(w.patterns, leakPattern{en: en, key: key, re: regexp.MustCompile("^" + strings.Join(quoted, ".+") + "$")}) |
| 314 | } |
| 315 | sort.Slice(w.patterns, func(i, j int) bool { return w.patterns[i].en < w.patterns[j].en }) |
| 316 | return w |
| 317 | } |
| 318 | |
| 319 | // leaks is the English a page in locale carries, given its runs, bar |
| 320 | // what leakUnchanged says the locale keeps. |
| 321 | func (w leakSweep) leaks(locale string, runs []string) []string { |
| 322 | var out []string |
| 323 | seen := map[string]bool{} |
| 324 | add := func(en string) { |
| 325 | if !seen[en] && !unchangedIn(locale, en) { |
| 326 | seen[en] = true |
| 327 | out = append(out, en) |
| 328 | } |
| 329 | } |
| 330 | words := strings.Join(runs, "\x00") |
| 331 | for _, s := range w.sentences { |
| 332 | if strings.Contains(words, s) { |
| 333 | add(s) |
| 334 | } |
| 335 | } |
| 336 | for _, r := range runs { |
| 337 | if w.whole[r] && !collidesIn(locale, r) { |
| 338 | add(r) |
| 339 | } |
| 340 | for _, p := range w.patterns { |
| 341 | if strings.Contains(r, p.key) && p.re.MatchString(r) { |
| 342 | add(p.en) |
| 343 | } |
| 344 | } |
| 345 | } |
| 346 | return out |
| 347 | } |
| 348 | |
| 349 | // leakSpoken are the attributes whose values a person reads or hears: |
| 350 | // Rastrillo's spoken attributes, alt, the other ARIA attributes that |
| 351 | // carry words, and the Copy live region's three words, which the |
| 352 | // script speaks. The page sweep and the template gate read the same |
| 353 | // list. |
| 354 | var leakSpoken = map[string]bool{ |
| 355 | "aria-label": true, "title": true, "placeholder": true, "label": true, "data-tip": true, "alt": true, |
| 356 | "data-copy": true, "data-copied": true, "data-failed": true, |
| 357 | "aria-description": true, "aria-roledescription": true, "aria-valuetext": true, "aria-placeholder": true, |
| 358 | } |
| 359 | |
| 360 | // leakSkipped says whether the sweep leaves an element and everything |
| 361 | // in it alone: script and style, which say nothing. The scheme menu |
| 362 | // and the HTML tab's source are swept like the rest, since the samples |
| 363 | // are in the page's language. |
| 364 | func leakSkipped(n *xhtml.Node) bool { |
| 365 | return n.Data == "script" || n.Data == "style" |
| 366 | } |
| 367 | |
| 368 | // leakKept says whether the sweep reads an element's spoken attributes |
| 369 | // and leaves its content alone: whatever the sample walker keeps as |
| 370 | // written (keptElements and keptAttrs: code, a drawing, a record's own |
| 371 | // words in <bdi>, a shell's brand and person, and anything that says |
| 372 | // its own language), which may well be words a sample elsewhere |
| 373 | // translates. The walker still translates the element's own spoken |
| 374 | // attributes, as it does every opening tag's, so the sweep reads them. |
| 375 | func leakKept(n *xhtml.Node) bool { |
| 376 | if keptElements[n.Data] { |
| 377 | return true |
| 378 | } |
| 379 | for _, a := range n.Attr { |
| 380 | // The walker reads bodies; a document's own <html lang> says |
| 381 | // the page's language, not that its words are kept. |
| 382 | if slices.Contains(keptAttrs, a.Key) && n.Data != "html" { |
| 383 | return true |
| 384 | } |
| 385 | } |
| 386 | return false |
| 387 | } |
| 388 | |
| 389 | // sourceOf says whether n is an HTML tab's source block, whose text is |
| 390 | // markup to be read as a sample is. |
| 391 | func sourceOf(n *xhtml.Node) bool { |
| 392 | if n.Data != "pre" { |
| 393 | return false |
| 394 | } |
| 395 | for _, a := range n.Attr { |
| 396 | if a.Key == "class" && slices.Contains(strings.Fields(a.Val), "ds-src") { |
| 397 | return true |
| 398 | } |
| 399 | } |
| 400 | return false |
| 401 | } |
| 402 | |
| 403 | // textContent is everything n's text nodes say, in order. |
| 404 | func textContent(n *xhtml.Node) string { |
| 405 | var b strings.Builder |
| 406 | var walk func(*xhtml.Node) |
| 407 | walk = func(n *xhtml.Node) { |
| 408 | if n.Type == xhtml.TextNode { |
| 409 | b.WriteString(n.Data) |
| 410 | } |
| 411 | for c := n.FirstChild; c != nil; c = c.NextSibling { |
| 412 | walk(c) |
| 413 | } |
| 414 | } |
| 415 | walk(n) |
| 416 | return b.String() |
| 417 | } |
| 418 | |
| 419 | // pageRuns is every run of text on a page and every spoken attribute |
| 420 | // value, as a reader gets them (entities decoded, whitespace |
| 421 | // collapsed), outside what leakSkipped leaves alone and the content |
| 422 | // of what leakKept keeps, empty runs left out; an HTML tab's source is |
| 423 | // read as the markup it shows. |
| 424 | func pageRuns(t *testing.T, doc []byte) []string { |
| 425 | t.Helper() |
| 426 | root, err := xhtml.Parse(strings.NewReader(string(doc))) |
| 427 | if err != nil { |
| 428 | t.Fatal(err) |
| 429 | } |
| 430 | var out []string |
| 431 | keep := func(s string) { |
| 432 | if s = squash(s); s != "" { |
| 433 | out = append(out, s) |
| 434 | } |
| 435 | } |
| 436 | var walk func(*xhtml.Node) |
| 437 | walk = func(n *xhtml.Node) { |
| 438 | switch n.Type { |
| 439 | case xhtml.ElementNode: |
| 440 | if leakSkipped(n) { |
| 441 | return |
| 442 | } |
| 443 | if sourceOf(n) { |
| 444 | // The HTML tab shows markup as text; it is read as the |
| 445 | // markup it is, so its words and spoken attributes are |
| 446 | // swept as a frame's are, and what the walker keeps is |
| 447 | // left as the walker left it. |
| 448 | out = append(out, pageRuns(t, []byte(textContent(n)))...) |
| 449 | return |
| 450 | } |
| 451 | for _, a := range n.Attr { |
| 452 | if leakSpoken[a.Key] { |
| 453 | keep(a.Val) |
| 454 | } |
| 455 | } |
| 456 | if leakKept(n) { |
| 457 | return |
| 458 | } |
| 459 | case xhtml.TextNode: |
| 460 | keep(n.Data) |
| 461 | } |
| 462 | for c := n.FirstChild; c != nil; c = c.NextSibling { |
| 463 | walk(c) |
| 464 | } |
| 465 | } |
| 466 | walk(root) |
| 467 | return out |
| 468 | } |
| 469 | |
| 470 | // pageWords is pageRuns joined with NULs, so a sentence found in it |
| 471 | // cannot span two runs. |
| 472 | func pageWords(t *testing.T, doc []byte) string { |
| 473 | t.Helper() |
| 474 | return strings.Join(pageRuns(t, doc), "\x00") |
| 475 | } |
| 476 | |
| 477 | // englishLeaks is which of sentences the page's words carry. |
| 478 | func englishLeaks(words string, sentences []string) []string { |
| 479 | var out []string |
| 480 | for _, s := range sentences { |
| 481 | if strings.Contains(words, s) { |
| 482 | out = append(out, s) |
| 483 | } |
| 484 | } |
| 485 | return out |
| 486 | } |
| 487 | |
| 488 | // translatedPages is every page, as documentKind classes them, of the |
| 489 | // non-English locales in the tree. |
| 490 | func translatedPages(s *site, files map[string][]byte) []string { |
| 491 | return translatedOf(s, files, docPage) |
| 492 | } |
| 493 | |
| 494 | // translatedDocuments is every document documentKind classes, page or |
| 495 | // sample document, of the non-English locales in the tree. |
| 496 | func translatedDocuments(s *site, files map[string][]byte) []string { |
| 497 | return translatedOf(s, files, docPage, docSample) |
| 498 | } |
| 499 | |
| 500 | // translatedOf is every document of the non-English locales whose |
| 501 | // documentKind is one of kinds. |
| 502 | func translatedOf(s *site, files map[string][]byte, kinds ...string) []string { |
| 503 | var out []string |
| 504 | for name := range files { |
| 505 | if slices.Contains(kinds, s.documentKind(name)) && !strings.Contains(name, "/en/") { |
| 506 | out = append(out, name) |
| 507 | } |
| 508 | } |
| 509 | sort.Strings(out) |
| 510 | return out |
| 511 | } |
| 512 | |
| 513 | // translationOf is locale's translation of a traced id: the sample |
| 514 | // catalog's for a sample, the catalog's for anything else. |
| 515 | func translationOf(t *testing.T, locale, id string) string { |
| 516 | t.Helper() |
| 517 | if k, ok := strings.CutPrefix(id, "samples."); ok { |
| 518 | return loadedSamples(t)[locale][k] |
| 519 | } |
| 520 | return loadedCatalogs(t)[locale][id] |
| 521 | } |
| 522 | |
| 523 | // TestNoEnglishProseReachesATranslatedPage: no English a render looks |
| 524 | // up, chrome, prose, label or sample, is in the text or the spoken |
| 525 | // attributes of any document in a language that is not English: every |
| 526 | // page, the lifted scheme menu and the HTML tab's source included, and |
| 527 | // every sample document, the frames, the demo application and the |
| 528 | // modal route. A sentence anywhere in a run, any string as a whole run, |
| 529 | // bar what leakUnchanged says a locale keeps. The controls are catalog |
| 530 | // values put back to their English, a sentence and two short words of |
| 531 | // the chrome and a sample's word, which the sweep must report on the |
| 532 | // document that says them. |
| 533 | func TestNoEnglishProseReachesATranslatedPage(t *testing.T) { |
| 534 | files := render(t, "") |
| 535 | s := newTestSite(t, fiesta.FS(), "") |
| 536 | english := englishStrings(t) |
| 537 | sweep := leakSweepFor(t, english) |
| 538 | if len(sweep.sentences) < 100 || len(sweep.whole) < 200 || len(sweep.patterns) < 5 { |
| 539 | t.Fatalf("sweeping for %d sentences, %d whole strings and %d patterns; the sweep has stopped reading the traced strings", len(sweep.sentences), len(sweep.whole), len(sweep.patterns)) |
| 540 | } |
| 541 | |
| 542 | // Every file under a theme and locale is one class or the other, |
| 543 | // so the sweep cannot quietly read fewer pages than there are. |
| 544 | for name := range files { |
| 545 | if strings.HasSuffix(name, ".html") && strings.Count(name, "/") >= 2 && s.documentKind(name) == "" { |
| 546 | t.Errorf("%s is neither a page nor a sample document", name) |
| 547 | } |
| 548 | } |
| 549 | // Each string leakUnchanged lets through is one a render says, and |
| 550 | // each locale it names does keep it, so the list cannot outlive the |
| 551 | // reason for an entry. |
| 552 | for en, locales := range leakUnchanged { |
| 553 | if locales == nil { |
| 554 | locales = nonEnglish() |
| 555 | } |
| 556 | kept := false |
| 557 | for id := range tracedIDs(t) { |
| 558 | if english[id] != en { |
| 559 | continue |
| 560 | } |
| 561 | for _, l := range locales { |
| 562 | if tr := translationOf(t, l, id); tr != en { |
| 563 | t.Errorf("leakUnchanged keeps %q in %s, whose catalog says %q for %s", en, l, tr, id) |
| 564 | } |
| 565 | } |
| 566 | kept = true |
| 567 | } |
| 568 | if !kept { |
| 569 | t.Errorf("leakUnchanged keeps %q, which no render says", en) |
| 570 | } |
| 571 | } |
| 572 | // Each word leakCollisions excuses is the English of a string a |
| 573 | // render says, and in each locale it names, the translation of |
| 574 | // another: the language's own word, spelt as English spells it. |
| 575 | for word, locales := range leakCollisions { |
| 576 | said := false |
| 577 | for id := range tracedIDs(t) { |
| 578 | said = said || squash(english[id]) == word |
| 579 | } |
| 580 | if !said { |
| 581 | t.Errorf("leakCollisions excuses %q, which is the English of nothing a render says", word) |
| 582 | } |
| 583 | for _, l := range locales { |
| 584 | own := false |
| 585 | for id := range tracedIDs(t) { |
| 586 | if squash(translationOf(t, l, id)) == word && squash(english[id]) != word { |
| 587 | own = true |
| 588 | } |
| 589 | } |
| 590 | if !own { |
| 591 | t.Errorf("leakCollisions excuses %q in %s, where it translates nothing; it is English there", word, l) |
| 592 | } |
| 593 | } |
| 594 | } |
| 595 | docs := translatedDocuments(s, files) |
| 596 | perDir := len(s.pageKinds()) + len(s.m.Examples) + len(s.extraDocumentNames()) |
| 597 | for _, e := range s.m.Examples { |
| 598 | perDir += len(s.frameFiles(e)) |
| 599 | } |
| 600 | if want := len(fiesta.ThemeNames()) * len(nonEnglish()) * perDir; len(docs) != want { |
| 601 | t.Errorf("sweeping %d translated documents, want %d", len(docs), want) |
| 602 | } |
| 603 | for _, name := range docs { |
| 604 | for _, en := range sweep.leaks(strings.Split(name, "/")[1], pageRuns(t, files[name])) { |
| 605 | t.Errorf("%s carries the English %q", name, en) |
| 606 | } |
| 607 | } |
| 608 | |
| 609 | // The controls: three strings put back to their English in the ja |
| 610 | // catalog, a sentence and two short words of the chrome (the Copy |
| 611 | // live region's word and the Overview's name), each of which the |
| 612 | // sweep must report on the Japanese Overview; and sample strings put |
| 613 | // back in their sample catalogs, each of which it must report on the |
| 614 | // document that shows it: a word of the sidebar shell, a short |
| 615 | // drafted link of the demo, and a carried sentence of a field's |
| 616 | // error. The sweep is built again with the catalogs changed, so no |
| 617 | // exemption it draws can excuse what a control put back. |
| 618 | cs := loadedCatalogs(t) |
| 619 | samples := loadedSamples(t) |
| 620 | controls := []string{"gallery.overview.intro", "gallery.copy", "gallery.kind.overview"} |
| 621 | sampleControls := []struct{ locale, key, doc string }{ |
| 622 | {"ja", "Sign out", "day/ja/frames/layout-sidebar/" + mustEntry(t, "layout-sidebar").Default + ".html"}, |
| 623 | {"ja", "← Requests", "day/ja/demo-request.html"}, |
| 624 | {"es", "A summary needs at least twenty characters.", "day/es/frames/field-textarea/with-an-error.html"}, |
| 625 | } |
| 626 | orig := map[string]string{} |
| 627 | for _, id := range controls { |
| 628 | orig[id] = cs["ja"][id] |
| 629 | cs["ja"][id] = text[id] |
| 630 | } |
| 631 | origSamples := make([]string, len(sampleControls)) |
| 632 | for i, c := range sampleControls { |
| 633 | origSamples[i] = samples[c.locale][c.key] |
| 634 | samples[c.locale][c.key] = c.key |
| 635 | } |
| 636 | scratch, err := Render(fiesta.FS(), Options{}) |
| 637 | controlSweep := leakSweepFor(t, english) |
| 638 | for _, id := range controls { |
| 639 | cs["ja"][id] = orig[id] |
| 640 | } |
| 641 | for i, c := range sampleControls { |
| 642 | samples[c.locale][c.key] = origSamples[i] |
| 643 | } |
| 644 | if err != nil { |
| 645 | t.Fatal(err) |
| 646 | } |
| 647 | found := controlSweep.leaks("ja", pageRuns(t, scratch["day/ja/index.html"])) |
| 648 | for _, id := range controls { |
| 649 | if !slices.Contains(found, squash(text[id])) { |
| 650 | t.Errorf("with %s put back to its English in the ja catalog, the sweep does not report %q on day/ja/index.html; it reports %q", id, text[id], found) |
| 651 | } |
| 652 | } |
| 653 | for i, c := range sampleControls { |
| 654 | if origSamples[i] == "" || origSamples[i] == c.key { |
| 655 | t.Errorf("the %s sample catalog says %q for %q; the control proves nothing", c.locale, origSamples[i], c.key) |
| 656 | } |
| 657 | if !strings.Contains(string(scratch[c.doc]), c.key) { |
| 658 | t.Errorf("%s does not show %q once it is put back; the control is on the wrong document", c.doc, c.key) |
| 659 | } |
| 660 | if found := controlSweep.leaks(c.locale, pageRuns(t, scratch[c.doc])); !slices.Contains(found, c.key) { |
| 661 | t.Errorf("with %q put back to its English in the %s sample catalog, the sweep does not report it on %s; it reports %q", c.key, c.locale, c.doc, found) |
| 662 | } |
| 663 | } |
| 664 | |
| 665 | // A kept element's own spoken attributes are the walker's to |
| 666 | // translate, as it translates every opening tag's, so the sweep |
| 667 | // reads them while it leaves the element's content alone. |
| 668 | runs := pageRuns(t, []byte(`<p><bdi aria-label="Help: orders">Grace Hopper</bdi> <a rst-shell-brand href="#" title="Back to the start">Harbour</a></p>`)) |
| 669 | for _, want := range []string{"Help: orders", "Back to the start"} { |
| 670 | if !slices.Contains(runs, want) { |
| 671 | t.Errorf("the sweep does not read the spoken attribute %q of a kept element; it reads %q", want, runs) |
| 672 | } |
| 673 | } |
| 674 | for _, kept := range []string{"Grace Hopper", "Harbour"} { |
| 675 | if slices.Contains(runs, kept) { |
| 676 | t.Errorf("the sweep reads %q, the content of a kept element", kept) |
| 677 | } |
| 678 | } |
| 679 | } |
| 680 | |
| 681 | // leakSweepFor is the sweep over a full render's traced strings. Its |
| 682 | // exemptions are leakExempt, leakUnchanged and leakCollisions, lists |
| 683 | // written down, so nothing a catalog says can widen them; the controls |
| 684 | // build it again after changing the catalogs all the same, so that |
| 685 | // stays true if an exemption is ever drawn from them. |
| 686 | func leakSweepFor(t *testing.T, english map[string]string) leakSweep { |
| 687 | t.Helper() |
| 688 | return newLeakSweep(tracedIDs(t), english) |
| 689 | } |
| 690 | |
| 691 | func asRendered(s string) string { |
| 692 | var b strings.Builder |
| 693 | if err := renderedText.Execute(&b, s); err != nil { |
| 694 | panic(err) |
| 695 | } |
| 696 | return b.String() |
| 697 | } |
| 698 | |
| 699 | var renderedText = template.Must(template.New("").Parse("{{.}}")) |
| 700 | |
| 701 | // escapedSentences are traced strings html/template writes escaped: one |
| 702 | // with an apostrophe, one with a quote and one with a plus sign, where |
| 703 | // the gallery says any, each a whole sentence the sweep looks for. |
| 704 | func escapedSentences(t *testing.T) map[string]string { |
| 705 | t.Helper() |
| 706 | english := englishStrings(t) |
| 707 | ids := tracedIDs(t) |
| 708 | sorted := make([]string, 0, len(ids)) |
| 709 | for id := range ids { |
| 710 | sorted = append(sorted, id) |
| 711 | } |
| 712 | sort.Strings(sorted) |
| 713 | out := map[string]string{} |
| 714 | for _, mark := range []string{"'", `"`, "+"} { |
| 715 | for _, id := range sorted { |
| 716 | en := english[id] |
| 717 | if strings.Contains(en, mark) && !placeholderName.MatchString(en) && len([]rune(en)) >= leakFloor && en == squash(en) && !strings.HasPrefix(id, "labels.") { |
| 718 | out[id] = en |
| 719 | break |
| 720 | } |
| 721 | } |
| 722 | } |
| 723 | if len(out) < 2 { |
| 724 | t.Fatalf("found %d escaped sentences to try, want at least the apostrophe and the quote", len(out)) |
| 725 | } |
| 726 | return out |
| 727 | } |
| 728 | |
| 729 | // TestTheLeakSweepSeesEscapedPunctuation: a sentence html/template |
| 730 | // writes with ', " or + is still reported. Each one, with |
| 731 | // its Irish translation put back to the English, is on the Irish pages |
| 732 | // only in its escaped form, and the sweep reports it there. |
| 733 | func TestTheLeakSweepSeesEscapedPunctuation(t *testing.T) { |
| 734 | s := newTestSite(t, fiesta.FS(), "") |
| 735 | cs := loadedCatalogs(t) |
| 736 | for id, en := range escapedSentences(t) { |
| 737 | if asRendered(en) == en { |
| 738 | t.Errorf("%s (%q) is written as it is, so it is not the case this covers", id, en) |
| 739 | } |
| 740 | if got := englishLeaks(pageWords(t, []byte("<p>"+asRendered(en)+"</p>")), []string{en}); len(got) != 1 { |
| 741 | t.Errorf("the rendered form of %q is not reported", en) |
| 742 | } |
| 743 | orig := cs["ga"][id] |
| 744 | cs["ga"][id] = en |
| 745 | files, err := Render(fiesta.FS(), Options{}) |
| 746 | cs["ga"][id] = orig |
| 747 | if err != nil { |
| 748 | t.Fatal(err) |
| 749 | } |
| 750 | var raw strings.Builder |
| 751 | reported := false |
| 752 | for _, name := range translatedPages(s, files) { |
| 753 | if !strings.Contains(name, "/ga/") { |
| 754 | continue |
| 755 | } |
| 756 | raw.Write(files[name]) |
| 757 | if len(englishLeaks(pageWords(t, files[name]), []string{en})) == 1 { |
| 758 | reported = true |
| 759 | } |
| 760 | } |
| 761 | if strings.Contains(raw.String(), en) { |
| 762 | t.Errorf("the raw English of %s is on the Irish pages, so this case does not reach the escaped form", id) |
| 763 | } |
| 764 | if !reported { |
| 765 | t.Errorf("with its Irish translation put back to the English, %s (%q) is not reported on any Irish page", id, en) |
| 766 | } |
| 767 | } |
| 768 | } |
| 769 | |
| 770 | // literalText is every run of letters a template writes outside its |
| 771 | // actions: the text between tags, an SVG's title and desc included, |
| 772 | // and the values of the spoken attributes, however they are quoted. |
| 773 | // The template is read by the HTML tokenizer with each action replaced |
| 774 | // by actionHole, a private-use character no template writes, and a run |
| 775 | // is cut at each hole: what an action prints is a lookup or a field. Script and style say nothing; an SVG's drawing data (d, |
| 776 | // points, viewBox) is in attributes that are not spoken, so it is |
| 777 | // never read. |
| 778 | func literalText(src string) []string { |
| 779 | z := xhtml.NewTokenizer(strings.NewReader(templateAction.ReplaceAllString(src, actionHole))) |
| 780 | var out []string |
| 781 | keep := func(chunk string) { |
| 782 | for _, piece := range strings.Split(chunk, actionHole) { |
| 783 | if c := squash(piece); c != "" && templateLetter.MatchString(c) { |
| 784 | out = append(out, c) |
| 785 | } |
| 786 | } |
| 787 | } |
| 788 | silent := false |
| 789 | for { |
| 790 | switch tt := z.Next(); tt { |
| 791 | case xhtml.ErrorToken: |
| 792 | return out |
| 793 | case xhtml.StartTagToken, xhtml.SelfClosingTagToken: |
| 794 | name, more := z.TagName() |
| 795 | for more { |
| 796 | var k, v []byte |
| 797 | k, v, more = z.TagAttr() |
| 798 | if leakSpoken[string(k)] { |
| 799 | keep(string(v)) |
| 800 | } |
| 801 | } |
| 802 | if tt == xhtml.StartTagToken && (string(name) == "script" || string(name) == "style") { |
| 803 | silent = true |
| 804 | } |
| 805 | case xhtml.EndTagToken: |
| 806 | if name, _ := z.TagName(); string(name) == "script" || string(name) == "style" { |
| 807 | silent = false |
| 808 | } |
| 809 | case xhtml.TextToken: |
| 810 | if !silent { |
| 811 | keep(string(z.Text())) |
| 812 | } |
| 813 | } |
| 814 | } |
| 815 | } |
| 816 | |
| 817 | var ( |
| 818 | // An action, non-greedy: a say call's arguments carry no braces. |
| 819 | templateAction = regexp.MustCompile(`(?s)\{\{.*?\}\}`) |
| 820 | templateLetter = regexp.MustCompile(`\p{L}`) |
| 821 | ) |
| 822 | |
| 823 | // actionHole stands where an action was. |
| 824 | const actionHole = "\ue000" |
| 825 | |
| 826 | // templateFixtures are the runs of letters a template may write as |
| 827 | // they are, because they are the same in every language: the type |
| 828 | // specimen beside each font-size token, as Rastrillo's list has it, |
| 829 | // and the two halves of the icon file path, icons/<slug>.svg, which |
| 830 | // is code a reader copies. Adding to it is a claim that a string is |
| 831 | // not a word. |
| 832 | var templateFixtures = map[string]bool{ |
| 833 | "Ag": true, |
| 834 | "icons/": true, |
| 835 | ".svg": true, |
| 836 | } |
| 837 | |
| 838 | // TestNoUnregisteredEnglishInThePageTemplates: every word the page, |
| 839 | // body and view templates put on a page comes out of an action, a |
| 840 | // {{say …}} or a field, so no English is written straight into a |
| 841 | // template where the catalogs cannot reach it; templateFixtures names |
| 842 | // the few runs that are not words. The demo's documents are not among |
| 843 | // these templates: their words are sample content. The controls are a |
| 844 | // set of scratch templates: literal words in text, in an SVG's title |
| 845 | // and desc, and in spoken attributes quoted either way or not at all, |
| 846 | // which the extractor finds, and words in a script or a style and SVG |
| 847 | // drawing data, which it does not. |
| 848 | func TestNoUnregisteredEnglishInThePageTemplates(t *testing.T) { |
| 849 | if len(templateSources) < 8 { |
| 850 | t.Fatalf("%d registered templates; the page, the view and the six bodies are eight", len(templateSources)) |
| 851 | } |
| 852 | fixtures := 0 |
| 853 | for i, src := range templateSources { |
| 854 | name := regexp.MustCompile(`\{\{define "([^"]+)"\}\}`).FindStringSubmatch(src) |
| 855 | for _, s := range literalText(src) { |
| 856 | if templateFixtures[s] { |
| 857 | fixtures++ |
| 858 | continue |
| 859 | } |
| 860 | t.Errorf("template %d (%v) writes %q literally; say it through {{say …}} and a copy.go key", i, name, s) |
| 861 | } |
| 862 | } |
| 863 | if fixtures != len(templateFixtures) { |
| 864 | t.Errorf("the templates write %d of the %d fixtures; a fixture no template writes any more is to be dropped", fixtures, len(templateFixtures)) |
| 865 | } |
| 866 | for _, c := range []struct { |
| 867 | src string |
| 868 | want []string |
| 869 | }{ |
| 870 | {`{{define "x"}}<p title="Hello">{{say "gallery.brand"}} there</p>{{end}}`, []string{"Hello", "there"}}, |
| 871 | {`<svg viewBox="0 0 24 24" aria-hidden="true"><title>Settings</title><desc>A cog</desc><path d="M12 2a10 10 0 1 0 0 20"/></svg>`, []string{"Settings", "A cog"}}, |
| 872 | {`<button aria-label='Close'>{{icon "x"}}</button><img alt=Logo src="{{.Src}}">`, []string{"Close", "Logo"}}, |
| 873 | {`<script>const label = "Hidden";</script><style>p::after { content: "Gone" }</style><p>{{.Name}}</p>`, nil}, |
| 874 | } { |
| 875 | if got := literalText(c.src); !slices.Equal(got, c.want) { |
| 876 | t.Errorf("the extractor reads %q from %s, want %q", got, c.src, c.want) |
| 877 | } |
| 878 | } |
| 879 | } |
| 880 | |
amadan