| 1 | // 🤖 The rich composer's document core (#531): schema, markdown round-trip, |
| 2 | // and doc⇄plaintext offset mapping. Pure — no EditorView, no DOM at module |
| 3 | // scope — so every rule here is Node-testable (test/composer.test.mjs). The |
| 4 | // view glue lives in composer-pm.js. |
| 5 | // |
| 6 | // Three load-bearing rules: |
| 7 | // |
| 8 | // 1. THE KEYBOARD NEVER SPLITS PARAGRAPHS. Enter-that-means-newline inserts a |
| 9 | // hard_break; paragraph nodes only arise from parsing "\n\n" on edit-load. |
| 10 | // The serializer maps hard_break → "\n" and block joins → "\n\n", so a |
| 11 | // plain multi-line message serializes byte-identical to what the textarea |
| 12 | // sent yesterday — the wire format must not drift under old clients. |
| 13 | // |
| 14 | // 2. ZERO-ESCAPE POLICY. Unmarked plain text serializes verbatim — never a |
| 15 | // backslash. Old fleet clients (TestFlight/Sparkle lag) render escapes |
| 16 | // literally, and "what you type is what sends" is today's behaviour. |
| 17 | // Escapes appear only where the serializer's own generated syntax would be |
| 18 | // corrupted: `*` inside em/strong runs, `~` inside strike, brackets in |
| 19 | // link labels. Consequence, accepted + tested: serialize∘parse is NOT the |
| 20 | // identity on markdown-shaped plain text (parse("- a") is a real list) — |
| 21 | // but serialize∘parse∘serialize = serialize always holds. |
| 22 | // |
| 23 | // 3. ONE PARSER ON WEB. The markdown-side parse uses markdown.js's |
| 24 | // houseParser() (the feed's exact configuration), so what the composer |
| 25 | // understands and what the feed renders can never disagree. The one |
| 26 | // deliberate delta: `image` is disabled here, so an edit-loaded |
| 27 | //  becomes "!" + a link — which serializes back to the same |
| 28 | // bytes — instead of an image node the schema doesn't have. |
| 29 | |
| 30 | import { Schema, MarkdownParser, MarkdownSerializer } from "./vendor/prosemirror.js"; |
| 31 | import { houseParser } from "./markdown.js"; |
| 32 | |
| 33 | // --- schema ------------------------------------------------------------------- |
| 34 | |
| 35 | // The feed's allowlist, minus what only exists at render time (mentions, |
| 36 | // autolinked bare URLs). No headings, no images, no tables, no hr. The link |
| 37 | // mark exists for edit-load round-trips and pasted rich text — typing a URL |
| 38 | // stays plain text (the feed autolinks at render, same as today). |
| 39 | export const composerSchema = new Schema({ |
| 40 | nodes: { |
| 41 | doc: { content: "block+" }, |
| 42 | paragraph: { |
| 43 | content: "inline*", |
| 44 | group: "block", |
| 45 | parseDOM: [{ tag: "p" }], |
| 46 | toDOM: () => ["p", 0], |
| 47 | }, |
| 48 | blockquote: { |
| 49 | content: "block+", |
| 50 | group: "block", |
| 51 | defining: true, |
| 52 | parseDOM: [{ tag: "blockquote" }], |
| 53 | toDOM: () => ["blockquote", 0], |
| 54 | }, |
| 55 | code_block: { |
| 56 | content: "text*", |
| 57 | marks: "", |
| 58 | group: "block", |
| 59 | code: true, |
| 60 | defining: true, |
| 61 | attrs: { params: { default: "" } }, |
| 62 | parseDOM: [{ tag: "pre", preserveWhitespace: "full" }], |
| 63 | toDOM: () => ["pre", ["code", 0]], |
| 64 | }, |
| 65 | bullet_list: { |
| 66 | content: "list_item+", |
| 67 | group: "block", |
| 68 | parseDOM: [{ tag: "ul" }], |
| 69 | toDOM: () => ["ul", 0], |
| 70 | }, |
| 71 | ordered_list: { |
| 72 | content: "list_item+", |
| 73 | group: "block", |
| 74 | attrs: { order: { default: 1 } }, |
| 75 | parseDOM: [ |
| 76 | { tag: "ol", getAttrs: (n) => ({ order: n.hasAttribute("start") ? +n.getAttribute("start") || 1 : 1 }) }, |
| 77 | ], |
| 78 | toDOM: (n) => (n.attrs.order === 1 ? ["ol", 0] : ["ol", { start: n.attrs.order }, 0]), |
| 79 | }, |
| 80 | list_item: { |
| 81 | content: "paragraph block*", |
| 82 | defining: true, |
| 83 | parseDOM: [{ tag: "li" }], |
| 84 | toDOM: () => ["li", 0], |
| 85 | }, |
| 86 | text: { group: "inline" }, |
| 87 | hard_break: { |
| 88 | inline: true, |
| 89 | group: "inline", |
| 90 | selectable: false, |
| 91 | parseDOM: [{ tag: "br" }], |
| 92 | toDOM: () => ["br"], |
| 93 | }, |
| 94 | }, |
| 95 | marks: { |
| 96 | em: { |
| 97 | parseDOM: [{ tag: "i" }, { tag: "em" }, { style: "font-style=italic" }], |
| 98 | toDOM: () => ["em", 0], |
| 99 | }, |
| 100 | strong: { |
| 101 | parseDOM: [ |
| 102 | { tag: "strong" }, |
| 103 | // Pasted Google-Docs-style bold comes as inline styles / <b>. |
| 104 | { tag: "b", getAttrs: (n) => n.style.fontWeight !== "normal" && null }, |
| 105 | { style: "font-weight", getAttrs: (v) => /^(bold(er)?|[5-9]\d{2,})$/.test(v) && null }, |
| 106 | ], |
| 107 | toDOM: () => ["strong", 0], |
| 108 | }, |
| 109 | strike: { |
| 110 | parseDOM: [{ tag: "s" }, { tag: "del" }, { tag: "strike" }, { style: "text-decoration=line-through" }], |
| 111 | toDOM: () => ["s", 0], |
| 112 | }, |
| 113 | code: { |
| 114 | excludes: "_", // code never combines with other marks, like the feed |
| 115 | parseDOM: [{ tag: "code" }], |
| 116 | toDOM: () => ["code", 0], |
| 117 | }, |
| 118 | link: { |
| 119 | attrs: { href: {} }, |
| 120 | inclusive: false, |
| 121 | parseDOM: [ |
| 122 | { |
| 123 | tag: "a[href]", |
| 124 | // Same gate as the feed's validateLink: http(s) or it isn't a link. |
| 125 | getAttrs: (n) => (/^https?:\/\//i.test(n.getAttribute("href") || "") ? { href: n.getAttribute("href") } : false), |
| 126 | }, |
| 127 | ], |
| 128 | toDOM: (n) => ["a", { href: n.attrs.href, rel: "noopener noreferrer" }, 0], |
| 129 | }, |
| 130 | }, |
| 131 | }); |
| 132 | |
| 133 | // --- markdown → doc ------------------------------------------------------------- |
| 134 | |
| 135 | const parserMd = houseParser(); |
| 136 | parserMd.disable("image"); // see the header note — "!"+link round-trips; an image node wouldn't |
| 137 | |
| 138 | const mdParser = new MarkdownParser(composerSchema, parserMd, { |
| 139 | paragraph: { block: "paragraph" }, |
| 140 | blockquote: { block: "blockquote" }, |
| 141 | bullet_list: { block: "bullet_list" }, |
| 142 | ordered_list: { |
| 143 | block: "ordered_list", |
| 144 | getAttrs: (tok) => ({ order: +tok.attrGet("start") || 1 }), |
| 145 | }, |
| 146 | list_item: { block: "list_item" }, |
| 147 | fence: { block: "code_block", getAttrs: (tok) => ({ params: (tok.info || "").trim() }), noCloseToken: true }, |
| 148 | // Every newline is a break in chat (feed parses breaks:true) — soft and |
| 149 | // hard breaks are the same node, and rule 1 keeps paragraphs out of typing. |
| 150 | softbreak: { node: "hard_break" }, |
| 151 | hardbreak: { node: "hard_break" }, |
| 152 | code_inline: { mark: "code", noCloseToken: true }, |
| 153 | em: { mark: "em" }, |
| 154 | strong: { mark: "strong" }, |
| 155 | s: { mark: "strike" }, |
| 156 | link: { mark: "link", getAttrs: (tok) => ({ href: tok.attrGet("href") || "" }) }, |
| 157 | }); |
| 158 | |
| 159 | export function parseToDoc(text) { |
| 160 | return mdParser.parse(String(text ?? "")); |
| 161 | } |
| 162 | |
| 163 | // --- doc → markdown ------------------------------------------------------------- |
| 164 | |
| 165 | // Rule 2: only escape where our own generated syntax would break. |
| 166 | function escapeInMarks(text, marks) { |
| 167 | const names = new Set(marks.map((m) => m.type.name)); |
| 168 | if (names.has("code")) return text; // delimiters are computed to clear the content |
| 169 | let out = text; |
| 170 | if (names.has("em") || names.has("strong")) out = out.replace(/\*/g, "\\*"); |
| 171 | if (names.has("strike")) out = out.replace(/~/g, "\\~"); |
| 172 | if (names.has("link")) out = out.replace(/[\[\]]/g, (c) => "\\" + c); |
| 173 | return out; |
| 174 | } |
| 175 | |
| 176 | // Inline-code delimiters sized past the content's longest backtick run, with |
| 177 | // CommonMark's space padding when the content itself starts/ends in a tick |
| 178 | // (the defaultMarkdownSerializer rule, replicated so `escape:false` is safe). |
| 179 | function backticksFor(node, side) { |
| 180 | const ticks = /`+/g; |
| 181 | let len = 0; |
| 182 | let m; |
| 183 | if (node.isText) while ((m = ticks.exec(node.text))) len = Math.max(len, m[0].length); |
| 184 | let result = len > 0 && side > 0 ? " `" : "`"; |
| 185 | for (let i = 0; i < len; i++) result += "`"; |
| 186 | if (len > 0 && side < 0) result += " "; |
| 187 | return result; |
| 188 | } |
| 189 | |
| 190 | // Literal-mode escaping for a plain run: marker characters everywhere, and |
| 191 | // block-forming syntax at line starts (so a typed "- " or "> " reads back as |
| 192 | // exactly those characters in every feed). `out` is the serializer's |
| 193 | // accumulated output — empty or newline-terminated means this run opens a line. |
| 194 | function escapeLiteralRun(text, out) { |
| 195 | let escaped = text.replace(/[\\`*_~]/g, (c) => "\\" + c); |
| 196 | const atLineStart = out === "" || out.endsWith("\n"); |
| 197 | const guardLine = (line) => |
| 198 | line |
| 199 | .replace(/^(\s*)([->+])(\s)/, "$1\\$2$3") |
| 200 | .replace(/^(\s*\d+)\.(\s)/, "$1\\.$2"); |
| 201 | if (atLineStart) escaped = guardLine(escaped); |
| 202 | return escaped; |
| 203 | } |
| 204 | |
| 205 | function safeHref(href) { |
| 206 | return /[\s()]/.test(href) ? "<" + href + ">" : href; |
| 207 | } |
| 208 | |
| 209 | const mdSerializer = new MarkdownSerializer( |
| 210 | { |
| 211 | paragraph(state, node) { |
| 212 | state.renderInline(node); |
| 213 | state.closeBlock(node); |
| 214 | }, |
| 215 | hard_break(state) { |
| 216 | state.write("\n"); // NOT markdown's "\\\n" — the wire format is plain newlines |
| 217 | }, |
| 218 | code_block(state, node) { |
| 219 | const runs = node.textContent.match(/`{3,}/g); |
| 220 | const fence = runs ? "`".repeat(Math.max(...runs.map((s) => s.length)) + 1) : "```"; |
| 221 | state.write(fence + (node.attrs.params || "") + "\n"); |
| 222 | state.text(node.textContent, false); |
| 223 | state.ensureNewLine(); |
| 224 | state.write(fence); |
| 225 | state.closeBlock(node); |
| 226 | }, |
| 227 | blockquote(state, node) { |
| 228 | state.wrapBlock("> ", null, node, () => state.renderContent(node)); |
| 229 | }, |
| 230 | bullet_list(state, node) { |
| 231 | state.renderList(node, " ", () => "- "); |
| 232 | }, |
| 233 | ordered_list(state, node) { |
| 234 | const start = node.attrs.order || 1; |
| 235 | const maxW = String(start + node.childCount - 1).length; |
| 236 | const space = " ".repeat(maxW + 2); |
| 237 | state.renderList(node, space, (i) => { |
| 238 | const nStr = String(start + i); |
| 239 | return " ".repeat(maxW - nStr.length) + nStr + ". "; |
| 240 | }); |
| 241 | }, |
| 242 | list_item(state, node) { |
| 243 | state.renderContent(node); |
| 244 | }, |
| 245 | text(state, node) { |
| 246 | // escapeLiterals (format-as-you-type OFF): plain-run marker characters |
| 247 | // are the member's literal text and must RENDER literally — escape |
| 248 | // them. Marked runs keep their generated markers (the WYSIWYG's work) |
| 249 | // and the normal in-mark escaping. |
| 250 | if (state.options.escapeLiterals && node.marks.length === 0) { |
| 251 | state.text(escapeLiteralRun(node.text, state.out), false); |
| 252 | } else { |
| 253 | state.text(escapeInMarks(node.text, node.marks), false); |
| 254 | } |
| 255 | }, |
| 256 | }, |
| 257 | { |
| 258 | em: { open: "*", close: "*", mixable: true, expelEnclosingWhitespace: true }, |
| 259 | strong: { open: "**", close: "**", mixable: true, expelEnclosingWhitespace: true }, |
| 260 | // Single tilde on purpose: the old grammar (still live across the fleet) |
| 261 | // renders ~s~ perfectly; ~~s~~ would show stray tildes there. |
| 262 | strike: { open: "~", close: "~", mixable: true, expelEnclosingWhitespace: true }, |
| 263 | link: { |
| 264 | open: "[", |
| 265 | close: (state, mark) => "](" + safeHref(mark.attrs.href) + ")", |
| 266 | }, |
| 267 | code: { |
| 268 | open: (state, mark, parent, index) => backticksFor(parent.child(index), -1), |
| 269 | close: (state, mark, parent, index) => backticksFor(parent.child(index - 1), 1), |
| 270 | escape: false, |
| 271 | }, |
| 272 | }, |
| 273 | ); |
| 274 | |
| 275 | export function serializeDoc(doc, options = {}) { |
| 276 | // tightLists: chat lists are always tight (the feed renders them tight, and |
| 277 | // loose serialization would inject \n\n old clients show as blank lines). |
| 278 | // The one escape prosemirror-markdown emits outside our handlers is the |
| 279 | // protective "\!" before a link (renderInline guards against accidental |
| 280 | // image syntax). Our zero-escape policy never writes "\!", so stripping it |
| 281 | // is unambiguous — and required: it's how an edit-loaded  |
| 282 | // round-trips byte-identical (the feed renders images literally anyway). |
| 283 | return mdSerializer |
| 284 | .serialize(doc, { tightLists: true, escapeLiterals: !!options.escapeLiterals }) |
| 285 | .replace(/\\(!\[)/g, "$1"); |
| 286 | } |
| 287 | |
| 288 | // --- doc ⇄ plaintext offsets ---------------------------------------------------- |
| 289 | // |
| 290 | // The typeahead contract (ui.js composerToken + its Swift twin) is plain |
| 291 | // string + caret offset. docText joins textblocks with "\n" and reads |
| 292 | // hard_break as "\n"; the two mappers translate offsets in that string to |
| 293 | // ProseMirror positions and back, walking in exactly docText's order. |
| 294 | |
| 295 | export function docText(doc) { |
| 296 | let out = ""; |
| 297 | let first = true; |
| 298 | doc.descendants((node) => { |
| 299 | if (!node.isTextblock) return true; |
| 300 | if (!first) out += "\n"; |
| 301 | first = false; |
| 302 | node.forEach((child) => { |
| 303 | out += child.isText ? child.text : "\n"; |
| 304 | }); |
| 305 | return false; |
| 306 | }); |
| 307 | return out; |
| 308 | } |
| 309 | |
| 310 | export function textOffsetFromPos(doc, pos) { |
| 311 | let off = 0; |
| 312 | let first = true; |
| 313 | let found = null; |
| 314 | doc.descendants((node, nodePos) => { |
| 315 | if (found !== null) return false; |
| 316 | if (!node.isTextblock) return true; |
| 317 | if (!first) off += 1; // the "\n" join before this block |
| 318 | first = false; |
| 319 | if (pos <= nodePos) { |
| 320 | found = off; |
| 321 | return false; |
| 322 | } |
| 323 | const contentEnd = nodePos + node.nodeSize - 1; |
| 324 | if (pos > contentEnd) { |
| 325 | node.forEach((child) => { |
| 326 | off += child.isText ? child.text.length : 1; |
| 327 | }); |
| 328 | return false; |
| 329 | } |
| 330 | let inner = nodePos + 1; |
| 331 | node.forEach((child) => { |
| 332 | if (found !== null) return; |
| 333 | if (pos <= inner) { |
| 334 | found = off; |
| 335 | return; |
| 336 | } |
| 337 | if (child.isText) { |
| 338 | if (pos <= inner + child.nodeSize) { |
| 339 | found = off + (pos - inner); |
| 340 | return; |
| 341 | } |
| 342 | off += child.text.length; |
| 343 | } else { |
| 344 | off += 1; // hard_break |
| 345 | } |
| 346 | inner += child.nodeSize; |
| 347 | }); |
| 348 | if (found === null) found = off; |
| 349 | return false; |
| 350 | }); |
| 351 | return found === null ? off : found; |
| 352 | } |
| 353 | |
| 354 | export function posFromTextOffset(doc, offset) { |
| 355 | let off = 0; |
| 356 | let first = true; |
| 357 | let result = null; |
| 358 | let lastEnd = 0; |
| 359 | doc.descendants((node, nodePos) => { |
| 360 | if (result !== null) return false; |
| 361 | if (!node.isTextblock) return true; |
| 362 | if (!first) { |
| 363 | // The join "\n" itself: an offset landing on it belongs to the START of |
| 364 | // this block (a caret after Enter sits before the next block's text). |
| 365 | off += 1; |
| 366 | } |
| 367 | first = false; |
| 368 | let inner = nodePos + 1; |
| 369 | node.forEach((child) => { |
| 370 | if (result !== null) return; |
| 371 | if (child.isText) { |
| 372 | if (offset <= off + child.text.length) { |
| 373 | result = inner + (offset - off); |
| 374 | return; |
| 375 | } |
| 376 | off += child.text.length; |
| 377 | } else { |
| 378 | if (offset <= off) { |
| 379 | result = inner; |
| 380 | return; |
| 381 | } |
| 382 | off += 1; // hard_break reads as "\n" |
| 383 | } |
| 384 | inner += child.nodeSize; |
| 385 | }); |
| 386 | if (result === null && offset <= off) result = inner; |
| 387 | lastEnd = inner; |
| 388 | return false; |
| 389 | }); |
| 390 | return result === null ? lastEnd : result; |
| 391 | } |
| 392 | |
amadan