rastrillo / fiesta Public

Clone
git clone https://amadan.net/rastrillo/fiesta

Plain git — no account needed to clone.

Download

Download this file

1// markup-spelling: old-spelling begin — see markup.go.
2
3package markup
4
5import (
6 "fmt"
7 "regexp"
8 "strings"
9)
10
11// Note is something the rewrite could not do and a human has to look
12// at. A codemod that guesses is worse than one that says where it
13// stopped, so an unrecognised shape is left exactly as it was and
14// reported here.
15type Note struct {
16 Line int // 1-based, in the file as it was read
17 Text string // what was found, and why it was left alone
18}
19
20func (n Note) String() string { return fmt.Sprintf("%d: %s", n.Line, n.Text) }
21
22var (
23 rstClassToken = regexp.MustCompile(`rst-[A-Za-z0-9_-]+`)
24 tmplAction = regexp.MustCompile(`\{\{[^{}]*\}\}`)
25)
26
27// Rewrite translates one file's markup from the class spelling to the
28// attribute spelling. It rewrites class attributes (both the plain
29// class="…" of a template and the class=\"…\" of a Go string literal)
30// and the data-tone attribute, and touches nothing else — a CSS
31// selector, a Go identifier, a comment that only names a class are all
32// left alone.
33//
34// It is idempotent. The output has no class attribute carrying a
35// migrating rst- name, so a second pass finds nothing to do.
36func Rewrite(src []byte) ([]byte, []Note) { return rewrite(src, true) }
37
38// Respell is Rewrite without the migration's renames: the pure grammar,
39// for markup that is already written in today's class vocabulary and
40// only wants the other spelling of it. It is what the stage-1 browser
41// drive translates its fixture with, where the two spellings must mean
42// the same thing rather than one being an upgrade of the other.
43func Respell(src []byte) ([]byte, []Note) { return rewrite(src, false) }
44
45func rewrite(src []byte, migrate bool) ([]byte, []Note) {
46 s := string(src)
47 var b strings.Builder
48 var notes []Note
49 note := func(at int, format string, args ...any) {
50 notes = append(notes, Note{
51 Line: 1 + strings.Count(s[:at], "\n"),
52 Text: fmt.Sprintf(format, args...),
53 })
54 }
55 fenced := fencedRegions(s)
56 i := 0
57 for {
58 a, ok := nextClassAttr(s, i)
59 if !ok {
60 break
61 }
62 // A region a human has fenced off: a page whose subject IS the
63 // old spelling, a fixture that has to be written in it. Without
64 // this the only opt-out is "do not run the tool on that file",
65 // which is not one — every repository migrating its own docs
66 // has a paragraph that must keep the spelling it is about.
67 if fenced(a.start) {
68 b.WriteString(s[i:a.end])
69 i = a.end
70 continue
71 }
72 value := s[a.valStart:a.valEnd]
73 if !strings.Contains(value, "rst-") {
74 b.WriteString(s[i:a.end])
75 i = a.end
76 continue
77 }
78 if a.unreadable {
79 note(a.start, "class=%q is a shape this cannot take apart — an unquoted value with a template "+
80 "action in it. Quote it, or rewrite it by hand: left as it was", value)
81 b.WriteString(s[i:a.end])
82 i = a.end
83 continue
84 }
85 if c, bad := notAClassList(value); bad {
86 // Almost always source that builds its markup by
87 // concatenation: the reader ran past the end of a string
88 // literal and what it is holding is an expression rather
89 // than a class list. Rewriting it would produce something
90 // that does not compile, so say where it is and move on.
91 note(a.start, "class=%q is not a class list — it holds %q, so this markup is probably built by "+
92 "concatenation and the reader ran past the end of a literal: left as it was", value, string(c))
93 b.WriteString(s[i:a.end])
94 i = a.end
95 continue
96 }
97 repl, err := rewriteClassValue(value, a.q, migrate)
98 if err != nil {
99 note(a.start, "%s", err.Error())
100 b.WriteString(s[i:a.end])
101 i = a.end
102 continue
103 }
104 // An attribute that translated to nothing at all takes its own
105 // leading space with it, or the tag keeps a gap where a class
106 // used to be.
107 start := a.start
108 if repl == "" {
109 for start > i && (s[start-1] == ' ' || s[start-1] == '\t') {
110 start--
111 }
112 }
113 b.WriteString(s[i:start])
114 b.WriteString(repl)
115 i = a.end
116 }
117 b.WriteString(s[i:])
118 // The fence is recomputed here rather than reused: the class pass
119 // moved every offset after its first rewrite, and the markers
120 // themselves are never rewritten, so they are still where they say.
121 body := b.String()
122 inFence := fencedRegions(body)
123 out := replaceOutsideFence(body, toneInMarkup, "${1}rst-tone=${2}", inFence)
124
125 // Escaped markup — a documentation page showing class="rst-box"
126 // as source. Rewriting it would mean deciding what the escaping is
127 // for; naming it is the honest half, and it is a shape that exists
128 // in any repository with migration notes of its own.
129 //
130 // The fence applies here as much as it does to the rewrite, and for
131 // longer: a rewrite you decline is over, but a note is a line in a
132 // report that stays until somebody deals with it, and the exit code
133 // stays at 3 while it does. A fenced region is somebody having
134 // already dealt with it. Without this, every file that fences off a
135 // paragraph about the old spelling reports work forever, and this
136 // repository was the proof: internal/markup's own files are fenced
137 // from line 1 and reported notes anyway, on every run, so
138 // `rastrillo markup .` could never exit 0 here on a clean tree. A
139 // report mode that always says there is work is what teaches people
140 // to skip it and run --fix.
141 //
142 // The regions are recomputed rather than reusing inFence: the tone
143 // pass above shortened the text at every replacement, so offsets
144 // after the first one no longer mean what they did.
145 noteFence := fencedRegions(out)
146 for _, m := range escapedClassAttr.FindAllStringSubmatchIndex(out, -1) {
147 if noteFence(m[0]) {
148 continue
149 }
150 if strings.Contains(out[m[2]:m[3]], "rst-") {
151 notes = append(notes, Note{
152 Line: 1 + strings.Count(out[:m[0]], "\n"),
153 Text: "escaped markup (" + out[m[0]:m[1]] + ") is documentation rather than markup: rewrite it by hand if it should teach the attribute spelling",
154 })
155 }
156 }
157 return []byte(out), notes
158}
159
160// replaceOutsideFence is regexp.ReplaceAllString that leaves fenced
161// regions alone.
162func replaceOutsideFence(s string, re *regexp.Regexp, repl string, fenced func(int) bool) string {
163 var b strings.Builder
164 last := 0
165 for _, m := range re.FindAllStringSubmatchIndex(s, -1) {
166 if fenced(m[0]) {
167 continue
168 }
169 b.WriteString(s[last:m[0]])
170 b.Write(re.ExpandString(nil, repl, s, m))
171 last = m[1]
172 }
173 b.WriteString(s[last:])
174 return b.String()
175}
176
177// The fence, and the only opt-out this tool has. A line carrying the
178// begin marker starts a region it will not rewrite; a line carrying the
179// end marker closes it. Deliberately one mechanism shared with the
180// framework's own spelling gate, so a repository has one thing to
181// learn and one thing to grep for.
182const (
183 FenceBegin = "markup-spelling: old-spelling begin"
184 // Assembled from halves, or this line would close the fence that
185 // the top of this file opens and the rest of the file would be
186 // rewritten by its own tool. The one place that has to be true.
187 FenceEnd = "markup-spelling: old-spelling " + "end"
188)
189
190// fencedRegions returns a predicate saying whether a byte offset falls
191// inside a fenced region.
192func fencedRegions(s string) func(int) bool {
193 type span struct{ from, to int }
194 var spans []span
195 open := -1
196 at := 0
197 for _, line := range strings.SplitAfter(s, "\n") {
198 switch {
199 case open < 0 && strings.Contains(line, FenceBegin):
200 open = at
201 case open >= 0 && strings.Contains(line, FenceEnd):
202 spans = append(spans, span{open, at + len(line)})
203 open = -1
204 }
205 at += len(line)
206 }
207 if open >= 0 {
208 spans = append(spans, span{open, len(s)})
209 }
210 return func(i int) bool {
211 for _, sp := range spans {
212 if i >= sp.from && i < sp.to {
213 return true
214 }
215 }
216 return false
217 }
218}
219
220// escapedClassAttr matches a class attribute written as escaped text —
221// what a page showing markup as source carries.
222var escapedClassAttr = regexp.MustCompile(`class=(?:&quot;|&#34;|&#x22;)([^&]*)(?:&quot;|&#34;|&#x22;)`)
223
224// classAttr is one class attribute found in the source: the span it
225// occupies, the span of its value, and the quoting a replacement's own
226// values must be written in.
227type classAttr struct {
228 start, end int
229 valStart, valEnd int
230 q string
231 // unreadable: this IS a class attribute, and its value is not
232 // something the grammar can take apart. Carried rather than
233 // silently skipped, because silence is the one thing this tool must
234 // not do about markup it has found.
235 unreadable bool
236}
237
238// nextClassAttr finds the next class attribute at or after from, in
239// every shape HTML allows: any case, whitespace around the =, and a
240// value that is double-quoted, single-quoted, escaped for a Go or
241// JavaScript string literal, or not quoted at all.
242//
243// It reads all of them rather than the one shape this framework happens
244// to write, because the alternative is the one thing this tool must
245// never do: tell an app whose templates use single quotes that there is
246// nothing to do, and let it find out at stage 3, when the class
247// selectors are gone and nothing says why the page is unstyled.
248func nextClassAttr(s string, from int) (classAttr, bool) {
249 for i := from; i < len(s); {
250 k := indexFold(s, "class", i)
251 if k < 0 {
252 return classAttr{}, false
253 }
254 i = k + len("class")
255 // Not the tail of a longer word: data-class, superclass.
256 if k > 0 && isIdentByte(s[k-1]) {
257 continue
258 }
259 j := skipSpace(s, i)
260 // An =, and not the head of ==, so a Go comparison is not markup.
261 if j >= len(s) || s[j] != '=' || (j+1 < len(s) && s[j+1] == '=') {
262 continue
263 }
264 lead := byte(' ')
265 if k > 0 {
266 lead = s[k-1]
267 }
268 a, ok := readAttrValue(s, skipSpace(s, j+1), lead)
269 if !ok {
270 continue
271 }
272 a.start = k
273 return a, true
274 }
275 return classAttr{}, false
276}
277
278// indexFold is strings.Index for an ASCII-lowercase needle, ignoring
279// case in the haystack.
280func indexFold(s, needle string, from int) int {
281 for i := from; i+len(needle) <= len(s); i++ {
282 if strings.EqualFold(s[i:i+len(needle)], needle) {
283 return i
284 }
285 }
286 return -1
287}
288
289func skipSpace(s string, i int) int {
290 for i < len(s) && (s[i] == ' ' || s[i] == '\t') {
291 i++
292 }
293 return i
294}
295
296func isIdentByte(c byte) bool {
297 return c == '-' || c == '_' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9'
298}
299
300// readAttrValue reads an attribute value that starts at k. The quote it
301// reports is the one a replacement writes its own values in; an
302// unquoted value becomes a double-quoted one, because a variant list
303// has a space in it and an unquoted attribute cannot hold one.
304func readAttrValue(s string, k int, lead byte) (classAttr, bool) {
305 closed := func(open, quote string) (classAttr, bool) {
306 valStart := k + len(open)
307 e := strings.Index(s[valStart:], quote)
308 if e < 0 {
309 return classAttr{}, false
310 }
311 return classAttr{
312 valStart: valStart,
313 valEnd: valStart + e,
314 end: valStart + e + len(quote),
315 q: quote,
316 }, true
317 }
318 switch {
319 case strings.HasPrefix(s[k:], `\"`):
320 return closed(`\"`, `\"`)
321 case strings.HasPrefix(s[k:], `\'`):
322 return closed(`\'`, `\'`)
323 case k < len(s) && s[k] == '"':
324 return closed(`"`, `"`)
325 case k < len(s) && s[k] == '\'':
326 return closed(`'`, `'`)
327 case unquotedAttrOK(lead) && k < len(s) && s[k] != '>' && s[k] != ' ' && s[k] != '\t' && s[k] != '\n':
328 e := k
329 for e < len(s) && s[e] != ' ' && s[e] != '\t' && s[e] != '\n' && s[e] != '>' && s[e] != '/' {
330 e++
331 }
332 switch classify(s[k:e]) {
333 case notMarkupAtAll:
334 return classAttr{}, false
335 case unreadableList:
336 return classAttr{valStart: k, valEnd: e, end: e, q: `"`, unreadable: true}, true
337 }
338 return classAttr{valStart: k, valEnd: e, end: e, q: `"`}, true
339 }
340 return classAttr{}, false
341}
342
343// unquotedAttrOK: an unquoted value is only read where an attribute can
344// actually begin. Without this, the "class=" of a URL query string
345// (?class=x&y=1) reads as markup and gets rewritten into nonsense.
346func unquotedAttrOK(lead byte) bool {
347 switch lead {
348 case ' ', '\t', '\n', '\r', '<', '`', '"', '\'', '(':
349 return true
350 }
351 return false
352}
353
354// The three things an unquoted value can be.
355const (
356 plainClassList = iota
357 // Not markup: a URL query (?class=x&y=1), or a class attribute
358 // written as escaped text (class=&quot;rst-box&quot;), which the
359 // escaped-markup pass reports instead. Silence is right for these.
360 notMarkupAtAll
361 // Markup, and unreadable: an unquoted value with a template action
362 // in it. Silence is wrong for this one — it is a class attribute
363 // carrying our vocabulary that will simply never be migrated.
364 unreadableList
365)
366
367func classify(value string) int {
368 if value == "" {
369 return notMarkupAtAll
370 }
371 for i := 0; i < len(value); i++ {
372 switch value[i] {
373 case '&', '?', '%', '#', '=', ';':
374 return notMarkupAtAll
375 case '{', '}':
376 return unreadableList
377 }
378 }
379 return plainClassList
380}
381
382// notAClassList reports the character that says a value is not an HTML
383// class list at all. None of them can appear in a class attribute — a
384// quote would have ended it — so finding one means the reader is
385// holding source rather than markup, which is what happens when the
386// markup is built by concatenating string literals.
387func notAClassList(value string) (byte, bool) {
388 for i := 0; i < len(value); i++ {
389 switch value[i] {
390 case '"', '\'', '`', '<', '>', '\\':
391 return value[i], true
392 }
393 }
394 return 0, false
395}
396
397// attr is one attribute being built: its name, the variant tokens the
398// literal class list gave it, and — for a class value that carries a
399// template action — the expression that supplies the rest of its value.
400type attr struct {
401 name string
402 variants []string
403 expr string
404 // When a conditional can produce no variant at all, the whole
405 // value — the = and the quotes with it — is written inside that
406 // conditional, so the attribute renders bare rather than empty:
407 // rst-person{{if .Large}}="lg"{{end}}. html/template reads an
408 // action in that position and both branches leave the parser in
409 // the same state, so it is a shape it can compile.
410 open, closes string
411}
412
413func rewriteClassValue(value, q string, migrate bool) (string, error) {
414 if !strings.Contains(value, "{{") {
415 kept, attrs := splitClassTokens(strings.Fields(value), migrate)
416 return render(kept, attrs, q), nil
417 }
418 return rewriteTemplatedClassValue(value, q, migrate)
419}
420
421// splitClassTokens sorts a literal class list into what stays in class
422// and what becomes an attribute, keeping first-appearance order so the
423// diff reads like the markup it replaces.
424func splitClassTokens(tokens []string, migrate bool) (kept []string, attrs []*attr) {
425 byName := map[string]*attr{}
426 for _, t := range tokens {
427 if !strings.HasPrefix(t, "rst-") {
428 kept = append(kept, t)
429 continue
430 }
431 name, variant, ok, drop := classFor(t, migrate)
432 if drop {
433 continue
434 }
435 if !ok {
436 kept = append(kept, t)
437 continue
438 }
439 a := byName[name]
440 if a == nil {
441 a = &attr{name: name}
442 byName[name] = a
443 attrs = append(attrs, a)
444 }
445 if variant != "" {
446 a.variants = append(a.variants, variant)
447 }
448 }
449 return kept, attrs
450}
451
452func render(kept []string, attrs []*attr, q string) string {
453 var parts []string
454 if len(kept) > 0 {
455 parts = append(parts, "class="+q+strings.Join(kept, " ")+q)
456 }
457 for _, a := range attrs {
458 if a.open != "" {
459 parts = append(parts, a.name+a.open+"="+q+a.expr+q+a.closes)
460 continue
461 }
462 value := strings.Join(a.variants, " ")
463 if a.expr != "" {
464 if value != "" {
465 value += " "
466 }
467 value += a.expr
468 }
469 if value == "" {
470 parts = append(parts, a.name)
471 continue
472 }
473 parts = append(parts, a.name+"="+q+value+q)
474 }
475 return strings.Join(parts, " ")
476}
477
478// rewriteTemplatedClassValue handles the one shape a class list takes
479// when a template writes part of it: a literal head, then a conditional
480// that adds a modifier of a kind the head already names.
481//
482// class="rst-badge{{with .Tone}} rst-badge--{{.}}{{end}}"
483// -> rst-badge{{with .Tone}}="{{.}}"{{end}}
484// class="rst-btn{{if .Danger}} rst-btn--danger{{else}} rst-btn--primary{{end}}"
485// -> rst-btn="{{if .Danger}}danger{{else}}primary{{end}}"
486//
487// Which of the two depends on whether the conditional can produce no
488// modifier at all. A branchless {{if}} or {{with}} can, so the = and
489// the quotes go inside it and the attribute renders bare — which is
490// what a reader copying the gallery should see. A choice between two
491// modifiers always produces one, so the conditional goes in the value,
492// where it reads as what it is.
493//
494// Anything else is left alone and reported. A class list whose
495// structure this cannot read is a handful of lines for a human, and a
496// wrong guess is a screen that renders unstyled.
497func rewriteTemplatedClassValue(value, q string, migrate bool) (string, error) {
498 head := value[:strings.Index(value, "{{")]
499 rest := value[len(head):]
500 kept, attrs := splitClassTokens(strings.Fields(head), migrate)
501
502 tokens := rstClassToken.FindAllString(rest, -1)
503 if len(tokens) == 0 {
504 return "", fmt.Errorf(`class=%q carries a template action and no rst- class inside it: left as it was`, value)
505 }
506 kind := ""
507 for _, t := range tokens {
508 k := strings.Index(t, "--")
509 if k < 0 {
510 return "", fmt.Errorf(`class=%q: %q inside a template action is not a modifier, so the value slot cannot carry it: left as it was`, value, t)
511 }
512 if kind == "" {
513 kind = t[:k]
514 } else if kind != t[:k] {
515 return "", fmt.Errorf(`class=%q names modifiers of two kinds (%s, %s) inside one template action: left as it was`, value, kind, t[:k])
516 }
517 }
518 // Everything in the conditional that is not an action must be one
519 // of those modifiers, or this is a shape the rule above does not
520 // describe.
521 known := map[string]bool{}
522 for _, t := range tokens {
523 known[t] = true
524 }
525 for _, f := range strings.Fields(tmplAction.ReplaceAllString(rest, " ")) {
526 if !known[f] {
527 return "", fmt.Errorf(`class=%q has literal text (%q) beside the modifiers in its template action: left as it was`, value, f)
528 }
529 }
530
531 name, _, ok, drop := classFor(kind, migrate)
532 if drop || !ok {
533 return "", fmt.Errorf(`class=%q builds a modifier of %s, which does not migrate: left as it was`, value, kind)
534 }
535 strip := regexp.MustCompile(`\s*` + regexp.QuoteMeta(kind+"--"))
536
537 a := (*attr)(nil)
538 for _, cand := range attrs {
539 if cand.name == name {
540 a = cand
541 }
542 }
543 if a == nil {
544 a = &attr{name: name}
545 attrs = append(attrs, a)
546 }
547
548 actions := tmplAction.FindAllString(rest, -1)
549 branchless := len(a.variants) == 0 &&
550 len(actions) >= 2 &&
551 strings.HasPrefix(actions[len(actions)-1], "{{end") &&
552 strings.HasSuffix(rest, actions[len(actions)-1])
553 for _, act := range actions {
554 if strings.HasPrefix(act, "{{else") {
555 branchless = false
556 }
557 }
558 if branchless {
559 a.open = actions[0]
560 a.closes = actions[len(actions)-1]
561 a.expr = strip.ReplaceAllString(rest[len(a.open):len(rest)-len(a.closes)], "")
562 return render(kept, attrs, q), nil
563 }
564 a.expr = strip.ReplaceAllString(rest, "")
565 return render(kept, attrs, q), nil
566}
567