| 1 | package export |
| 2 | |
| 3 | import ( |
| 4 | "strconv" |
| 5 | "strings" |
| 6 | ) |
| 7 | |
| 8 | // Markdown renders a document as CommonMark. |
| 9 | // |
| 10 | // §11.3 says "Markdown lossless within its subset", and this is where |
| 11 | // that subset is drawn: headings, paragraphs, bullet and ordered lists |
| 12 | // (nested), blockquotes, and the bold/italic/code marks. What falls |
| 13 | // outside it is the font and size marks — Markdown has no way to say |
| 14 | // "this run is in the sans face at the large size", and inventing an |
| 15 | // HTML span to carry it would produce a file that is no longer |
| 16 | // Markdown for the many readers who would then see the tag. A person |
| 17 | // exporting to Markdown is asking for the words and their structure; |
| 18 | // the typography is what PDF is for. |
| 19 | // |
| 20 | // Paragraph alignment falls outside it too, and this says so out loud |
| 21 | // rather than letting it go missing quietly. CommonMark has no syntax |
| 22 | // for alignment at all — not a weaker spelling of it, none — so a |
| 23 | // centred paragraph exports as its own text, byte-identical to the same |
| 24 | // paragraph unaligned, and writeBlock's paragraph case never reads |
| 25 | // `align`. The alternatives fail the same way the font marks' would: an |
| 26 | // inline `<p align="center">` or a `style` attribute is HTML, not |
| 27 | // Markdown, and every reader that does not render raw HTML would show |
| 28 | // the tag; a `:---:` alignment row would silently turn one paragraph |
| 29 | // into a table. The words and the structure survive, which is the |
| 30 | // promise; the layout is what the PDF is for. |
| 31 | // |
| 32 | // The document title becomes an H1 at the top. It lives on the |
| 33 | // database row rather than inside the CRDT, so it is not in the tree, |
| 34 | // and a Markdown file that opens with nothing but the first paragraph |
| 35 | // loses the one piece of a document a person is most likely to be |
| 36 | // looking for in a folder of them. |
| 37 | func Markdown(doc Document) string { |
| 38 | var b strings.Builder |
| 39 | if doc.Title != "" { |
| 40 | b.WriteString("# ") |
| 41 | b.WriteString(escapeMarkdown(doc.Title)) |
| 42 | b.WriteString("\n\n") |
| 43 | } |
| 44 | writeBlocks(&b, doc.Root.Content, "") |
| 45 | // Exactly one trailing newline: blocks each end with a blank line, |
| 46 | // which leaves two. |
| 47 | return strings.TrimRight(b.String(), "\n") + "\n" |
| 48 | } |
| 49 | |
| 50 | // hardBreak is how a Shift+Enter line break (core/src/fragment.rs's |
| 51 | // `hard_break`, `<br>` in the read view) is spelled here: a backslash |
| 52 | // at the end of the line. |
| 53 | // |
| 54 | // CommonMark has two spellings and they are not equally safe. Two |
| 55 | // trailing spaces mean the same thing to a parser and nothing at all |
| 56 | // to a person reading the file, and every editor, formatter and |
| 57 | // pre-commit hook that trims trailing whitespace deletes them without |
| 58 | // anyone noticing — at which point the break the author typed has |
| 59 | // quietly become a soft wrap, which is the failure this whole fix |
| 60 | // exists to stop. A backslash is visible in the source, nothing strips |
| 61 | // it, and it cannot be mistaken for one the author typed: escapeMarkdown |
| 62 | // escapes that to `\\`. So this is the form that round-trips. |
| 63 | const hardBreak = "\\\n" |
| 64 | |
| 65 | // writeBlocks renders a run of block-level nodes at one indent level. |
| 66 | // indent is the prefix every line of a nested block carries — that is |
| 67 | // how a list inside a list stays inside it in the output. |
| 68 | func writeBlocks(b *strings.Builder, nodes []Node, indent string) { |
| 69 | for _, n := range nodes { |
| 70 | writeBlock(b, n, indent) |
| 71 | } |
| 72 | } |
| 73 | |
| 74 | func writeBlock(b *strings.Builder, n Node, indent string) { |
| 75 | switch kindOf(n) { |
| 76 | case blockHeading: |
| 77 | b.WriteString(indent) |
| 78 | b.WriteString(strings.Repeat("#", headingLevel(n))) |
| 79 | b.WriteString(" ") |
| 80 | // A heading is one line and Markdown has no way to spell a |
| 81 | // second one, so a break inside one becomes a space. The |
| 82 | // alternative is a backslash that strands the rest of the |
| 83 | // heading as a paragraph below it — the words are what has to |
| 84 | // survive, which is this file's rule everywhere else too. |
| 85 | b.WriteString(strings.ReplaceAll(inlineString(n.Content), hardBreak, " ")) |
| 86 | b.WriteString("\n\n") |
| 87 | |
| 88 | case blockParagraph: |
| 89 | text := inlineString(n.Content) |
| 90 | // A break at the end of a paragraph has no line to continue |
| 91 | // onto, and the backslash it would leave behind is not a break |
| 92 | // in CommonMark at all — it is a literal backslash appearing in |
| 93 | // a document whose author never typed one. TrimSuffix in a loop |
| 94 | // rather than TrimRight: TrimRight works on a character set, so |
| 95 | // it would also eat the second half of an escaped backslash the |
| 96 | // author DID type ("\\" from escapeMarkdown) and leave a |
| 97 | // dangling escape behind. |
| 98 | for strings.HasSuffix(text, hardBreak) { |
| 99 | text = strings.TrimSuffix(text, hardBreak) |
| 100 | } |
| 101 | if strings.TrimSpace(text) == "" { |
| 102 | // An empty paragraph is a blank line in the document and |
| 103 | // nothing at all in Markdown — two blank lines in a row |
| 104 | // are one paragraph break, not two. Emitting it would |
| 105 | // change nothing about how the file reads and would make |
| 106 | // the round trip look lossy when it is not. |
| 107 | return |
| 108 | } |
| 109 | b.WriteString(indent) |
| 110 | // A hard break puts a second line inside this one paragraph, |
| 111 | // and that line is still inside whatever block the paragraph |
| 112 | // is in — so it carries the same prefix. (writeList and the |
| 113 | // blockquote case re-prefix the lines they are given, so |
| 114 | // indent is "" underneath either of them; this is what keeps |
| 115 | // the stray-list_item path below honest.) |
| 116 | b.WriteString(strings.ReplaceAll(text, "\n", "\n"+indent)) |
| 117 | b.WriteString("\n\n") |
| 118 | |
| 119 | case blockBulletList: |
| 120 | writeList(b, n, indent, false) |
| 121 | |
| 122 | case blockOrderedList: |
| 123 | writeList(b, n, indent, true) |
| 124 | |
| 125 | case blockQuote: |
| 126 | // Render the quoted content into its own buffer and then |
| 127 | // prefix every line, so a multi-paragraph or list-containing |
| 128 | // quote is quoted throughout rather than only on its first |
| 129 | // line. |
| 130 | var inner strings.Builder |
| 131 | writeBlocks(&inner, n.Content, "") |
| 132 | for _, line := range strings.Split(strings.TrimRight(inner.String(), "\n"), "\n") { |
| 133 | b.WriteString(indent) |
| 134 | if line == "" { |
| 135 | b.WriteString(">\n") |
| 136 | continue |
| 137 | } |
| 138 | b.WriteString("> ") |
| 139 | b.WriteString(line) |
| 140 | b.WriteString("\n") |
| 141 | } |
| 142 | b.WriteString("\n") |
| 143 | |
| 144 | case blockListItem: |
| 145 | // Reached only when a list_item turns up outside a list, which |
| 146 | // ProseMirror's schema cannot produce but a client posting raw |
| 147 | // Yjs bytes can. Rendered as its own content rather than |
| 148 | // dropped — the module rule the core states and this file |
| 149 | // inherits: never lose the words. |
| 150 | writeBlocks(b, n.Content, indent) |
| 151 | |
| 152 | default: |
| 153 | if n.IsText() { |
| 154 | // A bare text run outside any block — same provenance as |
| 155 | // the stray list_item above. Its own paragraph. |
| 156 | b.WriteString(indent) |
| 157 | writeInline(b, []Node{n}) |
| 158 | b.WriteString("\n\n") |
| 159 | return |
| 160 | } |
| 161 | // An element this build does not know: render its children. |
| 162 | // A newer editor's node must not take a reader's text with it. |
| 163 | writeBlocks(b, n.Content, indent) |
| 164 | } |
| 165 | } |
| 166 | |
| 167 | func writeList(b *strings.Builder, list Node, indent string, ordered bool) { |
| 168 | for i, item := range list.Content { |
| 169 | marker := "- " |
| 170 | if ordered { |
| 171 | marker = strconv.Itoa(i+1) + ". " |
| 172 | } |
| 173 | // Continuation lines line up under the text, not under the |
| 174 | // marker, or a nested list reads as a sibling of the one above |
| 175 | // it. |
| 176 | childIndent := indent + strings.Repeat(" ", len(marker)) |
| 177 | |
| 178 | var inner strings.Builder |
| 179 | writeBlocks(&inner, item.Content, "") |
| 180 | lines := strings.Split(strings.TrimRight(inner.String(), "\n"), "\n") |
| 181 | for j, line := range lines { |
| 182 | switch { |
| 183 | case j == 0: |
| 184 | b.WriteString(indent) |
| 185 | b.WriteString(marker) |
| 186 | b.WriteString(line) |
| 187 | case line == "": |
| 188 | // Keep the blank line, but without trailing spaces. |
| 189 | default: |
| 190 | b.WriteString(childIndent) |
| 191 | b.WriteString(line) |
| 192 | } |
| 193 | b.WriteString("\n") |
| 194 | } |
| 195 | } |
| 196 | b.WriteString("\n") |
| 197 | } |
| 198 | |
| 199 | func inlineString(nodes []Node) string { |
| 200 | var b strings.Builder |
| 201 | writeInline(&b, nodes) |
| 202 | return b.String() |
| 203 | } |
| 204 | |
| 205 | // writeInline renders a run of inline nodes with their marks. |
| 206 | // |
| 207 | // Marks are applied per run rather than merged across adjacent runs |
| 208 | // that share one: "**bold**" followed by "**er**" reads identically to |
| 209 | // "**bolder**" once rendered, and merging would mean tracking mark |
| 210 | // state across a sequence for no visible gain. |
| 211 | func writeInline(b *strings.Builder, nodes []Node) { |
| 212 | for _, n := range nodes { |
| 213 | if n.Type == "hard_break" { |
| 214 | b.WriteString(hardBreak) |
| 215 | // A break has no children through the editor's schema and |
| 216 | // can have them through raw Yjs bytes; render them rather |
| 217 | // than lose them, exactly as fragment.rs does. |
| 218 | writeInline(b, n.Content) |
| 219 | continue |
| 220 | } |
| 221 | if !n.IsText() { |
| 222 | // An inline element (or a block that ended up inline). |
| 223 | // Render its children so nothing is lost. |
| 224 | writeInline(b, n.Content) |
| 225 | continue |
| 226 | } |
| 227 | if n.Text == "" { |
| 228 | continue |
| 229 | } |
| 230 | if n.Has("code") { |
| 231 | // Code spans take no other markup and no escaping — the |
| 232 | // backticks are the escape. A run containing a backtick |
| 233 | // needs a longer fence than any run of backticks inside |
| 234 | // it, which is what CommonMark says and what this does. |
| 235 | b.WriteString(codeSpan(n.Text)) |
| 236 | continue |
| 237 | } |
| 238 | // Leading and trailing spaces have to sit outside the |
| 239 | // delimiters: "** bold **" is not emphasis in CommonMark, it |
| 240 | // is four literal asterisks and a word. |
| 241 | lead, core, trail := splitSpace(n.Text) |
| 242 | if core == "" { |
| 243 | b.WriteString(n.Text) |
| 244 | continue |
| 245 | } |
| 246 | var opening, closing string |
| 247 | if n.Has("strong") { |
| 248 | opening, closing = opening+"**", "**"+closing |
| 249 | } |
| 250 | if n.Has("em") { |
| 251 | opening, closing = opening+"*", "*"+closing |
| 252 | } |
| 253 | b.WriteString(lead) |
| 254 | b.WriteString(opening) |
| 255 | b.WriteString(escapeMarkdown(core)) |
| 256 | b.WriteString(closing) |
| 257 | b.WriteString(trail) |
| 258 | } |
| 259 | } |
| 260 | |
| 261 | // splitSpace divides s into its leading whitespace, its middle, and |
| 262 | // its trailing whitespace. |
| 263 | func splitSpace(s string) (lead, core, trail string) { |
| 264 | core = strings.TrimLeft(s, " \t") |
| 265 | lead = s[:len(s)-len(core)] |
| 266 | trimmed := strings.TrimRight(core, " \t") |
| 267 | trail = core[len(trimmed):] |
| 268 | return lead, trimmed, trail |
| 269 | } |
| 270 | |
| 271 | // codeSpan wraps s in enough backticks to contain it. |
| 272 | func codeSpan(s string) string { |
| 273 | longest, run := 0, 0 |
| 274 | for _, r := range s { |
| 275 | if r == '`' { |
| 276 | run++ |
| 277 | if run > longest { |
| 278 | longest = run |
| 279 | } |
| 280 | } else { |
| 281 | run = 0 |
| 282 | } |
| 283 | } |
| 284 | fence := strings.Repeat("`", longest+1) |
| 285 | // A code span that starts or ends with a backtick needs a space |
| 286 | // inside the fence, which CommonMark strips again on the way back. |
| 287 | pad := "" |
| 288 | if strings.HasPrefix(s, "`") || strings.HasSuffix(s, "`") { |
| 289 | pad = " " |
| 290 | } |
| 291 | return fence + pad + s + pad + fence |
| 292 | } |
| 293 | |
| 294 | // escapeMarkdown backslash-escapes the characters that would otherwise |
| 295 | // turn a person's own words into markup. A document containing |
| 296 | // "2 * 3 * 4" must not come back as emphasis, and one containing |
| 297 | // "# 1 priority" must not come back as a heading. |
| 298 | // |
| 299 | // The set is CommonMark's ASCII punctuation, minus the characters that |
| 300 | // are only special in positions this function's callers never place |
| 301 | // text in. It escapes more than the strict minimum on purpose: an |
| 302 | // over-escaped file renders correctly and reads slightly noisily in |
| 303 | // source, while an under-escaped one renders as something the author |
| 304 | // did not write. |
| 305 | // "The start of a line" is tracked rather than inferred from index 0, |
| 306 | // and it means the first NON-SPACE on a line rather than the first |
| 307 | // character. |
| 308 | // |
| 309 | // Two rounds of this were wrong, in the same direction, and both are |
| 310 | // worth naming because the second looked like a fix. |
| 311 | // |
| 312 | // It first asked whether i == 0, which is the start of the RUN and not |
| 313 | // the start of a line. A run can contain a literal newline — nothing |
| 314 | // stops one: POST /d/{sqid}/updates accepts Yjs bytes that never met |
| 315 | // ProseMirror's schema, and a paste can carry one — so everything |
| 316 | // after a newline was treated as mid-sentence, and "#" became a |
| 317 | // heading, "- " a bullet, "1." an ordered list, mid-paragraph. |
| 318 | // |
| 319 | // Tracking lineStart fixed exactly the unindented case and no more. |
| 320 | // CommonMark lets a block marker carry up to THREE spaces of |
| 321 | // indentation (§4.2 for ATX headings, §5.2 for lists) and all of them |
| 322 | // interrupt a paragraph, so " # x" one space in is still a heading |
| 323 | // while `i == lineStart` says it is mid-line. Caught in review, with |
| 324 | // the seven inputs now in the test table. |
| 325 | // |
| 326 | // So the state is "have we seen anything but spaces on this line yet", |
| 327 | // which is what atLineStart means below. Any run of spaces is treated |
| 328 | // as still-at-line-start rather than counting to three: over-escaping |
| 329 | // renders correctly and reads slightly noisily in source, which is the |
| 330 | // trade this function's own header already chose. |
| 331 | // |
| 332 | // Two characters are escaped that a first reading would not expect: |
| 333 | // |
| 334 | // - '=' because a line of them is a setext heading, which underlines |
| 335 | // the paragraph ABOVE it — so it silently promotes text that is |
| 336 | // nowhere near it. ('-' is already escaped for the bullet case, |
| 337 | // which covers setext H2 as a side effect.) |
| 338 | // - '~' because "~~~" opens a code fence that interrupts a paragraph |
| 339 | // and then swallows every following line until a closing fence. It |
| 340 | // does not restyle a line, it eats the rest of the document. |
| 341 | // |
| 342 | // Both only at the start of a line, so "a = b" is left alone. |
| 343 | // |
| 344 | // KNOWN AND NOT FIXED HERE: four or more leading spaces after a |
| 345 | // newline make an indented code block, and no amount of escaping |
| 346 | // changes that — the block is made by the whitespace, not by any |
| 347 | // character this function could put a backslash in front of. The text |
| 348 | // survives verbatim and is rendered as code. Fixing it means altering |
| 349 | // the author's own leading whitespace, which is a different decision |
| 350 | // from escaping and not one to take in passing. |
| 351 | // TestMarkdownIndentedRunLinesBecomeCodeBlocks records the current |
| 352 | // behaviour so it stays a decision rather than becoming a discovery. |
| 353 | func escapeMarkdown(s string) string { |
| 354 | var b strings.Builder |
| 355 | b.Grow(len(s)) |
| 356 | lineStart := 0 // byte index where the current line begins |
| 357 | atLineStart := true // nothing but spaces seen on this line yet |
| 358 | for i, r := range s { |
| 359 | switch r { |
| 360 | case '\n': |
| 361 | b.WriteRune(r) |
| 362 | lineStart = i + 1 |
| 363 | atLineStart = true |
| 364 | continue |
| 365 | case ' ', '\t': |
| 366 | // Indentation does not end the line's start. |
| 367 | b.WriteRune(r) |
| 368 | continue |
| 369 | case '\\', '`', '*', '_', '[', ']', '<', '>': |
| 370 | b.WriteByte('\\') |
| 371 | b.WriteRune(r) |
| 372 | case '#', '+', '-', '=', '~': |
| 373 | // Only special at the start of a line. Inside a sentence |
| 374 | // they are ordinary punctuation, and escaping every hyphen |
| 375 | // in a document would be unreadable. |
| 376 | if atLineStart { |
| 377 | b.WriteByte('\\') |
| 378 | } |
| 379 | b.WriteRune(r) |
| 380 | case '.', ')': |
| 381 | // "1." and "1)" start an ordered list, but only after |
| 382 | // digits at the start of a line — so the digits are counted |
| 383 | // from the line's first non-space, not from the run. |
| 384 | if !atLineStart && startsWithDigits(strings.TrimLeft(s[lineStart:i], " \t")) { |
| 385 | b.WriteByte('\\') |
| 386 | } |
| 387 | b.WriteRune(r) |
| 388 | default: |
| 389 | b.WriteRune(r) |
| 390 | } |
| 391 | atLineStart = false |
| 392 | } |
| 393 | return b.String() |
| 394 | } |
| 395 | |
| 396 | // startsWithDigits reports whether s is entirely ASCII digits — the |
| 397 | // "1" in a "1." that would start a list. |
| 398 | func startsWithDigits(s string) bool { |
| 399 | if s == "" { |
| 400 | return false |
| 401 | } |
| 402 | for i := 0; i < len(s); i++ { |
| 403 | if s[i] < '0' || s[i] > '9' { |
| 404 | return false |
| 405 | } |
| 406 | } |
| 407 | return true |
| 408 | } |
| 409 | |
amadan