| 1 | // Generates the ProseMirror-shaped Y.XmlFragment fixtures that define the |
| 2 | // wire contract for core/src/fragment.rs. These are the contract: never |
| 3 | // edit a fixture to make a test pass. A deliberate rule change regenerates |
| 4 | // them in the same commit. |
| 5 | import * as Y from 'yjs' |
| 6 | import { writeFileSync, mkdirSync } from 'node:fs' |
| 7 | |
| 8 | // The client id is pinned so the bytes are reproducible. Options:: |
| 9 | // with_guid_and_client_id on the Rust side pins the same pair; nothing |
| 10 | // in this pipeline may draw entropy (see the plan's Global Constraints). |
| 11 | function docWith(build) { |
| 12 | const doc = new Y.Doc() |
| 13 | doc.clientID = 1 |
| 14 | build(doc.getXmlFragment('prosemirror')) |
| 15 | return Y.encodeStateAsUpdate(doc) |
| 16 | } |
| 17 | |
| 18 | function el(name, attrs, ...children) { |
| 19 | const e = new Y.XmlElement(name) |
| 20 | for (const [k, v] of Object.entries(attrs ?? {})) e.setAttribute(k, v) |
| 21 | if (children.length) e.insert(0, children) |
| 22 | return e |
| 23 | } |
| 24 | |
| 25 | function text(s, marks) { |
| 26 | const t = new Y.XmlText() |
| 27 | t.insert(0, s, marks ?? undefined) |
| 28 | return t |
| 29 | } |
| 30 | |
| 31 | mkdirSync('vectors/fragment', { recursive: true }) |
| 32 | |
| 33 | writeFileSync('vectors/fragment/simple.bin', docWith(f => { |
| 34 | f.insert(0, [ |
| 35 | el('paragraph', null, text('First paragraph.')), |
| 36 | el('paragraph', null, text('Second paragraph.')), |
| 37 | ]) |
| 38 | })) |
| 39 | |
| 40 | writeFileSync('vectors/fragment/marks.bin', docWith(f => { |
| 41 | f.insert(0, [ |
| 42 | el('paragraph', null, |
| 43 | text('Plain then '), text('bold', { strong: {} }), |
| 44 | text(' then '), text('italic', { em: {} }), text('.')), |
| 45 | ]) |
| 46 | })) |
| 47 | |
| 48 | writeFileSync('vectors/fragment/nested.bin', docWith(f => { |
| 49 | f.insert(0, [ |
| 50 | el('heading', { level: '2' }, text('A heading')), |
| 51 | el('bullet_list', null, |
| 52 | el('list_item', null, el('paragraph', null, text('One'))), |
| 53 | el('list_item', null, el('paragraph', null, text('Two')))), |
| 54 | ]) |
| 55 | })) |
| 56 | |
| 57 | // styled.bin covers what the formatting bar can produce that the three |
| 58 | // fixtures above cannot: the font and size marks (core/src/fragment.rs's |
| 59 | // MARKS table), a run carrying two marks at once, and the two block |
| 60 | // shapes — blockquote and ordered_list — that had no fixture before the |
| 61 | // export paths needed to render them. The marks are attribute-free on |
| 62 | // purpose; see that table's own comment for why a mark never carries a |
| 63 | // value. |
| 64 | writeFileSync('vectors/fragment/styled.bin', docWith(f => { |
| 65 | f.insert(0, [ |
| 66 | el('paragraph', null, |
| 67 | text('sans', { font_sans: {} }), |
| 68 | text(' mono', { font_mono: {} }), |
| 69 | text(' large', { size_lg: {} }), |
| 70 | text(' both', { font_sans: {}, strong: {} })), |
| 71 | el('blockquote', null, el('paragraph', null, text('Quoted.'))), |
| 72 | el('ordered_list', null, |
| 73 | el('list_item', null, el('paragraph', null, text('First'))), |
| 74 | el('list_item', null, el('paragraph', null, text('Second')))), |
| 75 | ]) |
| 76 | })) |
| 77 | |
| 78 | // fonts.bin covers the font marks on their own, and exists because |
| 79 | // serif became sayable. Until sans took the default over, "serif" was |
| 80 | // spelled as the ABSENCE of a mark — there was no font_serif — so the |
| 81 | // vocabulary styled.bin pins had no way to express the face most |
| 82 | // documents were actually in. This fixture pins the whole of it: an |
| 83 | // unmarked run, which carries no span whatever the default happens to |
| 84 | // be, and one run per face. |
| 85 | // |
| 86 | // A fixture of its own, rather than three more runs inside styled.bin. |
| 87 | // These vectors are the wire contract, not an output of it: styled.bin's |
| 88 | // bytes are what a dozen assertions in core/tests/fragment.rs and |
| 89 | // internal/docs already compare against, and regenerating it would |
| 90 | // quietly change what every one of them means. Adding a vector changes |
| 91 | // nothing that already exists. |
| 92 | writeFileSync('vectors/fragment/fonts.bin', docWith(f => { |
| 93 | f.insert(0, [ |
| 94 | el('paragraph', null, |
| 95 | text('Unmarked'), |
| 96 | text(' serif', { font_serif: {} }), |
| 97 | text(' sans', { font_sans: {} }), |
| 98 | text(' mono', { font_mono: {} })), |
| 99 | ]) |
| 100 | })) |
| 101 | |
| 102 | // aligned.bin covers the first block-level formatting: a paragraph's |
| 103 | // `align` attribute. Unlike a mark, alignment belongs to the block, so |
| 104 | // it is a real Yjs XML attribute on the element — which is exactly why |
| 105 | // it needs a fixture of its own. y-prosemirror stores node attributes |
| 106 | // this way (createTypeFromElementNode: setAttribute for every non-null |
| 107 | // attr), so this is what an aligned document ACTUALLY contains on the |
| 108 | // wire, and core/src/fragment.rs has to read it back from there rather |
| 109 | // than from anyone's description of it. |
| 110 | // |
| 111 | // The first paragraph carries no attribute at all, which is the whole |
| 112 | // point of left being the default: the editor's schema stores null, |
| 113 | // y-prosemirror skips null, and an unaligned document is byte-identical |
| 114 | // to one written before alignment existed. The last one says "left" out |
| 115 | // loud, which a paste or a future build can produce and which must |
| 116 | // render as no class rather than as a class of its own. |
| 117 | writeFileSync('vectors/fragment/aligned.bin', docWith(f => { |
| 118 | f.insert(0, [ |
| 119 | el('paragraph', null, text('Left by default.')), |
| 120 | el('paragraph', { align: 'center' }, text('Centred.')), |
| 121 | el('paragraph', { align: 'right' }, text('Right.')), |
| 122 | el('paragraph', { align: 'left' }, text('Left, said out loud.')), |
| 123 | ]) |
| 124 | })) |
| 125 | |
| 126 | // indented.bin covers the second block attribute, and it exists as its |
| 127 | // own vector for a reason aligned.bin did not have: indent's value is a |
| 128 | // NUMBER, where align's is a string from a closed set of three. |
| 129 | // |
| 130 | // That difference is the thing this fixture is here to pin. A string |
| 131 | // attribute can only be handled one way — look it up in a table, drop |
| 132 | // anything not in it — so aligned.bin's job was to prove the table is |
| 133 | // consulted. A number has to be parsed and clamped instead, which is a |
| 134 | // second mechanism with a second set of ways to be wrong, and the |
| 135 | // wrong-looking cases are exactly the ones no editor will ever produce: |
| 136 | // a level past the maximum, a zero, a value that is not a number at |
| 137 | // all. POST /d/{sqid}/updates takes raw Yjs bytes, so every one of them |
| 138 | // is reachable by a client that never went through the schema, and the |
| 139 | // renderer's output goes into the page unescaped. |
| 140 | // |
| 141 | // So the last three paragraphs here are not decoration. "99" must |
| 142 | // render clamped to the deepest level rather than as docs-indent-99; |
| 143 | // "0" must render as no class at all, the same as no attribute; |
| 144 | // "3; drop" must fail to parse and render flush left, never reaching |
| 145 | // attribute position. A build that interpolated the attribute straight |
| 146 | // into a class name would pass every test written against the first |
| 147 | // three paragraphs and fail these. |
| 148 | // |
| 149 | // The first paragraph carries no attribute, for aligned.bin's own |
| 150 | // reason: an un-indented document must be byte-identical to one written |
| 151 | // before indent existed. |
| 152 | // |
| 153 | // The heading is here because headings are indentable and are not |
| 154 | // alignable — a real asymmetry in the schema, and one that a fixture |
| 155 | // covering only paragraphs would let a renderer get wrong in silence. |
| 156 | writeFileSync('vectors/fragment/indented.bin', docWith(f => { |
| 157 | f.insert(0, [ |
| 158 | el('paragraph', null, text('Flush left, no attribute.')), |
| 159 | el('paragraph', { indent: '1' }, text('One step.')), |
| 160 | el('paragraph', { indent: '8' }, text('The deepest step.')), |
| 161 | el('heading', { level: '2', indent: '2' }, text('An indented heading.')), |
| 162 | el('paragraph', { align: 'center', indent: '3' }, text('Centred and indented.')), |
| 163 | el('paragraph', { indent: '99' }, text('Past the maximum; clamps.')), |
| 164 | el('paragraph', { indent: '0' }, text('Zero is no indent.')), |
| 165 | el('paragraph', { indent: '3; drop' }, text('Not a number at all.')), |
| 166 | // The four values JavaScript's Number() accepts and a Rust parse |
| 167 | // does not. They are here because the two sides disagreed about |
| 168 | // every one of them for a review cycle — the editor rendered them |
| 169 | // indented and the read view rendered them flush left — and the |
| 170 | // agreement is now the thing being pinned rather than the clamp. |
| 171 | el('paragraph', { indent: '400' }, text('Too large for a u8; still clamps.')), |
| 172 | el('paragraph', { indent: ' 3' }, text('Leading space is not a number.')), |
| 173 | el('paragraph', { indent: '3.0' }, text('Nor is a decimal point.')), |
| 174 | el('paragraph', { indent: '0x3' }, text('Nor is hex.')), |
| 175 | ]) |
| 176 | })) |
| 177 | |
| 178 | // sizes.bin covers the point ladder, and the three legacy marks |
| 179 | // alongside it. |
| 180 | // |
| 181 | // styled.bin already pins size_sm/lg/xl, so this could have been read as |
| 182 | // redundant. It is not: the legacy marks are pinned there as the sizes |
| 183 | // this editor WRITES, and here as the sizes it must go on UNDERSTANDING |
| 184 | // after it stopped writing them. A change that dropped them from MARKS |
| 185 | // would leave styled.bin's assertions to be "fixed" by regenerating it; |
| 186 | // this fixture states the property directly, so the same change fails |
| 187 | // with a message about documents in the wild. |
| 188 | // |
| 189 | // The mixed run at the end is the case neither renderer can produce and |
| 190 | // POST /d/{sqid}/updates can: a run carrying a ladder mark AND a legacy |
| 191 | // one. Both this build and internal/export/pdf.go resolve it the same |
| 192 | // way — ladder first, in a fixed order — because a document must not |
| 193 | // measure one size on screen and another on paper, and an arbitrary but |
| 194 | // stable answer is what determinism needs. |
| 195 | writeFileSync('vectors/fragment/sizes.bin', docWith(f => { |
| 196 | f.insert(0, [ |
| 197 | el('paragraph', null, |
| 198 | text('Body'), |
| 199 | text(' eight', { size_8: {} }), |
| 200 | text(' fourteen', { size_14: {} }), |
| 201 | text(' seventy-two', { size_72: {} })), |
| 202 | el('paragraph', null, |
| 203 | text('Small', { size_sm: {} }), |
| 204 | text(' Large', { size_lg: {} }), |
| 205 | text(' Extra', { size_xl: {} })), |
| 206 | el('paragraph', null, |
| 207 | text('Both at once', { size_lg: {}, size_24: {} })), |
| 208 | ]) |
| 209 | })) |
| 210 | |
| 211 | // --- trap-containment vectors --- |
| 212 | // |
| 213 | // core/tests/wasm_trap_containment.rs mutation-fuzzes real fixtures and |
| 214 | // checks a control document's content survives untouched, reading it |
| 215 | // back through the wasm ABI's docs_core_text_len/docs_core_text_read. |
| 216 | // Those now read fragment_text (root "prosemirror"), not text_content |
| 217 | // (root "content") — see core/src/abi.rs's text_string doc comment — |
| 218 | // so that test needs fragment-shaped fixtures of its own rather than |
| 219 | // fixtures/data's Y.Text ones (which read back as "" through that ABI |
| 220 | // now). These four mirror fixtures/data's hello/unicode/formatted/ |
| 221 | // concurrent in content and purpose (plain ASCII, surrogate pairs and |
| 222 | // combining marks, inline formatting, a concurrent merge) so the same |
| 223 | // coverage that fuzzing wants — plain text, non-ASCII byte lengths, |
| 224 | // format-op decoding, CRDT merge structure — survives the move, just |
| 225 | // wrapped in a single top-level <paragraph> each instead of a bare |
| 226 | // Y.Text. |
| 227 | // |
| 228 | // plainTextOf mirrors core/src/fragment.rs's fragment_text: walk the |
| 229 | // tree, concatenate each Y.XmlText's own rendering, and join top-level |
| 230 | // block texts with a blank line. |
| 231 | // |
| 232 | // Uses each Y.XmlText's own toString(), not toDelta() — verified |
| 233 | // against the real Rust output, not assumed: fragment_text calls |
| 234 | // yrs's XmlTextRef::get_string(txn), and yrs's own get_string embeds |
| 235 | // formatting as inline pseudo-tags exactly the way Y.XmlText.toString() |
| 236 | // does in JS ("plain <strong>bold</strong> plain", not "plain bold |
| 237 | // plain") — this is XmlText's own "plain text" convention in both |
| 238 | // implementations, not the HTML fragment_html builds via its own MARKS |
| 239 | // table. A first version of this generator used toDelta() (mark-free) |
| 240 | // and it disagreed with the real Rust output on the very first run — |
| 241 | // core/tests/wasm_trap_containment.rs caught it immediately, which is |
| 242 | // the whole reason this fixture set exists. |
| 243 | function plainTextOf(doc) { |
| 244 | const f = doc.getXmlFragment('prosemirror') |
| 245 | const isBlock = (name) => |
| 246 | ['paragraph', 'heading', 'list_item', 'blockquote'].includes(name) |
| 247 | const blocks = [] |
| 248 | let cur = '' |
| 249 | function walk(node) { |
| 250 | if (node instanceof Y.XmlText) { |
| 251 | cur += node.toString() |
| 252 | } else if (node instanceof Y.XmlElement) { |
| 253 | for (const child of node.toArray()) walk(child) |
| 254 | if (isBlock(node.nodeName) && cur.length > 0) { |
| 255 | blocks.push(cur) |
| 256 | cur = '' |
| 257 | } |
| 258 | } |
| 259 | } |
| 260 | for (const child of f.toArray()) walk(child) |
| 261 | if (cur.length > 0) blocks.push(cur) |
| 262 | return blocks.join('\n\n') |
| 263 | } |
| 264 | |
| 265 | // emitTrap writes both halves a fixture needs: the update bytes, and |
| 266 | // (unlike docWith's three vectors above, which core/tests/fragment.rs |
| 267 | // asserts against with hand-written literals) an expected.json next to |
| 268 | // it, the same shape fixtures/generate.mjs writes for fixtures/data — |
| 269 | // wasm_trap_containment.rs reads that "text" field the same way |
| 270 | // wire_compat.rs's fixture() does. |
| 271 | function emitTrap(name, build) { |
| 272 | const doc = new Y.Doc() |
| 273 | doc.clientID = 1 |
| 274 | build(doc) |
| 275 | const update = Y.encodeStateAsUpdate(doc) |
| 276 | writeFileSync(`vectors/fragment/${name}.update.bin`, Buffer.from(update)) |
| 277 | writeFileSync( |
| 278 | `vectors/fragment/${name}.expected.json`, |
| 279 | JSON.stringify({ text: plainTextOf(doc) }, null, 2) + '\n', |
| 280 | ) |
| 281 | console.log(`${name}: ${update.length} bytes`) |
| 282 | } |
| 283 | |
| 284 | emitTrap('hello', (doc) => { |
| 285 | doc.getXmlFragment('prosemirror').insert(0, [el('paragraph', null, text('hello world'))]) |
| 286 | }) |
| 287 | |
| 288 | emitTrap('unicode', (doc) => { |
| 289 | // Surrogate pairs and combining marks: Yjs indexes UTF-16 code units and |
| 290 | // Rust indexes UTF-8 bytes, same reasoning as fixtures/generate.mjs's |
| 291 | // own 'unicode' fixture. |
| 292 | doc.getXmlFragment('prosemirror').insert(0, [el('paragraph', null, text('héllo 🌍 café'))]) |
| 293 | }) |
| 294 | |
| 295 | emitTrap('formatted', (doc) => { |
| 296 | // Inline formatting: exercises format-op decoding under mutation |
| 297 | // fuzzing, same reasoning as fixtures/generate.mjs's own 'formatted' |
| 298 | // fixture. plainTextOf strips marks via toDelta, so the expected text |
| 299 | // is just the concatenated runs regardless of the strong attribute. |
| 300 | doc.getXmlFragment('prosemirror').insert(0, [ |
| 301 | el('paragraph', null, text('plain '), text('bold', { strong: {} }), text(' plain')), |
| 302 | ]) |
| 303 | }) |
| 304 | |
| 305 | emitTrap('concurrent', (doc) => { |
| 306 | // Two docs each insert their own top-level paragraph before syncing, |
| 307 | // then merge — order-independent CRDT merge structure, same |
| 308 | // reasoning as fixtures/generate.mjs's own 'concurrent' fixture, at |
| 309 | // the block level rather than mid-run since Y.XmlFragment's children |
| 310 | // are blocks, not characters. |
| 311 | const a = new Y.Doc() |
| 312 | a.clientID = 1 |
| 313 | const b = new Y.Doc() |
| 314 | b.clientID = 2 |
| 315 | // doc.clientID (set to 3 here) overrides the emitTrap assignment |
| 316 | // above, same as generate.mjs's own concurrent fixture. |
| 317 | doc.clientID = 3 |
| 318 | a.getXmlFragment('prosemirror').insert(0, [el('paragraph', null, text('AAA'))]) |
| 319 | b.getXmlFragment('prosemirror').insert(0, [el('paragraph', null, text('BBB'))]) |
| 320 | Y.applyUpdate(a, Y.encodeStateAsUpdate(b)) |
| 321 | Y.applyUpdate(b, Y.encodeStateAsUpdate(a)) |
| 322 | Y.applyUpdate(doc, Y.encodeStateAsUpdate(a)) |
| 323 | }) |
| 324 | |
| 325 | console.log('wrote 6 fragment vectors + 4 trap-containment vectors') |
| 326 | |
amadan