amadan

oficina / docs Public

The core reports structure, and learns fonts and sizes

Two additions to core/src/fragment.rs, both in service of export.

fragment_json is a third renderer over the same walk: the document as
ProseMirror-shaped JSON. Markdown and PDF both need to know where a list
ends and which run is bold, and neither can get that from the HTML
renderer without a parser Go's standard library does not have. The ABI
gains docs_core_json_len/_read and goes to v3 — adding an export is
backwards compatible to the wasm, but internal/core's version check is
exact, so the bump turns "unknown export, deep inside a request" into a
refusal at load.

The MARKS table gains font_sans, font_mono, size_sm, size_lg and size_xl,
rendered as spans with fixed class names. Every one is attribute-free:
Store.HTML returns template.HTML because fragment.rs escapes every text
run AND puts no byte from a document into attribute position, and
POST /d/{sqid}/updates accepts arbitrary Yjs bytes that never went
through ProseMirror's schema. A mark carrying its own value would spend
that property. Serif and normal carry no mark — they are what a document
is when nobody has said otherwise.

mark_names sorts the marks it does not know. Attrs is a HashMap, so
without that the same document renders differently between two calls in
one process, which would make every golden file on the Go side flake
rather than fail.

internal/core's textOn/htmlOn became one readOn, and Text/HTML/JSON three
one-line wrappers over one read. It was two copies of the same forty
lines about to become three — three places for a fix like fix-round-1's
finding 3 (read the written count, not the buffer size) to be applied to
two of.
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01YYB9fUAfxXNzkeC3fsFYzR
Paul Campbell pushed by paul@keymail.dev cc0a2d2c5c75fc670798fb0ae265306cd220e611 parent fd1d21a
9 files changed, +716 −105
  • core/src/abi.rs +48 −1
  • core/src/fragment.rs +204 −10
  • core/src/lib.rs +16 −3
  • core/tests/fragment.rs +148 −2
  • docs/superpowers/plans/2026-08-31-formatting-and-export.md +208 −0
  • fixtures/generate-fragment.mjs +22 −1
  • fixtures/vectors/fragment/styled.bin binary
  • internal/core/core.go +70 −88
  • internal/core/testdata/docs_core.wasm binary
diff --git a/core/src/abi.rs b/core/src/abi.rs
index 1b16cdd..9901174 100644
--- a/core/src/abi.rs
+++ b/core/src/abi.rs
@@ -46,7 +46,7 @@
//! held a stale handle is exactly the kind of silent corruption §5.2
//! exists to avoid.
-use crate::{apply_update, fragment_html, fragment_text, CoreError};
+use crate::{apply_update, fragment_html, fragment_json, fragment_text, CoreError};
use std::cell::RefCell;
use std::collections::HashMap;
use yrs::{Doc, Options};
@@ -148,6 +148,17 @@ pub fn html_string(handle: u32) -> Option<String> {
get_doc(handle).map(|doc| fragment_html(&doc))
}
+/// `None` for an unknown handle — distinct from `Some(String::new())`, a
+/// real empty document (Finding 3), exactly as `text_string` and
+/// `html_string` above. Note that an empty document is never the empty
+/// *string* here: `fragment_json` always emits at least
+/// `{"type":"doc","content":[]}`, so a zero length from
+/// `docs_core_json_len` is unreachable for a live handle and the host
+/// need not treat it as a special case the way the other two do.
+pub fn json_string(handle: u32) -> Option<String> {
+ get_doc(handle).map(|doc| fragment_json(&doc))
+}
+
// --- wasm exports ---
// Amended by pre-flight ruling R2. The obvious version — Vec::with_capacity
@@ -360,3 +371,39 @@ pub unsafe extern "C" fn docs_core_html_read(handle: u32, out_ptr: u32, out_len:
// larger, so clamp explicitly rather than rely on that coincidence.
s.len().try_into().unwrap_or(i32::MAX)
}
+
+/// Returns the byte length of `handle`'s JSON rendering, or `ERR_NO_DOC`
+/// if `handle` doesn't name a live document — distinct from 0, exactly
+/// as `docs_core_text_len` and `docs_core_html_len` above. (0 is not
+/// reachable here at all for a live handle; see `json_string`.)
+#[no_mangle]
+pub extern "C" fn docs_core_json_len(handle: u32) -> i32 {
+ match json_string(handle) {
+ Some(s) => s.len().try_into().unwrap_or(i32::MAX),
+ None => ERR_NO_DOC,
+ }
+}
+
+/// # Safety
+///
+/// `out_ptr` and `out_len` must describe a byte range that lies entirely
+/// within this module's linear memory and is valid for writes for the
+/// duration of this call. (Checked by `in_bounds` before any write — this
+/// safety requirement covers what that check cannot see, such as
+/// concurrent mutation.)
+#[no_mangle]
+pub unsafe extern "C" fn docs_core_json_read(handle: u32, out_ptr: u32, out_len: u32) -> i32 {
+ let s = match json_string(handle) {
+ Some(s) => s,
+ None => return ERR_NO_DOC, // stale/unknown handle, not an empty read
+ };
+ if s.len() > out_len as usize {
+ return ERR_BUFFER_TOO_SMALL;
+ }
+ if !in_bounds(out_ptr, out_len) {
+ return ERR_INVALID_LENGTH;
+ }
+ std::ptr::copy_nonoverlapping(s.as_ptr(), out_ptr as *mut u8, s.len());
+ // Same clamp, same reason as the two read functions above.
+ s.len().try_into().unwrap_or(i32::MAX)
+}
diff --git a/core/src/fragment.rs b/core/src/fragment.rs
index fdf01f0..8c7145f 100644
--- a/core/src/fragment.rs
+++ b/core/src/fragment.rs
@@ -35,30 +35,60 @@ fn escape(s: &str, out: &mut String) {
}
}
-/// Marks this build knows, in a fixed order so output is deterministic.
-/// A mark not listed here contributes no wrapper but never suppresses
-/// its text.
-const MARKS: &[(&str, &str)] = &[("strong", "strong"), ("em", "em"), ("code", "code")];
+/// Marks this build knows, in a fixed order so output is deterministic:
+/// `(mark name, opening markup, closing markup)`. A mark not listed here
+/// contributes no wrapper but never suppresses its text.
+///
+/// Every one of these is attribute-free on the CRDT side — the mark's
+/// presence is the whole of its meaning — and the markup each one emits
+/// is a compile-time constant chosen by lookup from this table. That is
+/// what keeps `Store.HTML`'s `template.HTML` return honest
+/// (`internal/docs/store.go`): `fragment_html` escapes every text run
+/// AND puts no byte from a document into attribute position, so the
+/// markup is safe to write out unescaped. A mark carrying a value —
+/// `link`'s `href`, or a `fontFamily` naming its own face — would break
+/// that, and `POST /d/{sqid}/updates` accepts arbitrary client-supplied
+/// Yjs bytes that never went through ProseMirror's schema, so the value
+/// would be genuinely arbitrary. Adding one is a real decision about
+/// attribute escaping and (for `href`) a scheme allowlist, not a row in
+/// this table.
+///
+/// The font and size marks are ordered ahead of the semantic ones so a
+/// run carrying both nests as `<span class="…"><strong>…</strong></span>`
+/// rather than the other way round. Serif and "normal" carry no mark at
+/// all: they are what a document is when nobody has said otherwise, and
+/// spending a mark on the default would put a span around most of the
+/// text in most documents.
+const MARKS: &[(&str, &str, &str)] = &[
+ ("font_sans", "<span class=\"docs-font-sans\">", "</span>"),
+ ("font_mono", "<span class=\"docs-font-mono\">", "</span>"),
+ ("size_sm", "<span class=\"docs-size-sm\">", "</span>"),
+ ("size_lg", "<span class=\"docs-size-lg\">", "</span>"),
+ ("size_xl", "<span class=\"docs-size-xl\">", "</span>"),
+ ("strong", "<strong>", "</strong>"),
+ ("em", "<em>", "</em>"),
+ ("code", "<code>", "</code>"),
+];
fn text_html<T: ReadTxn>(txn: &T, t: &XmlTextRef, out: &mut String) {
// Diff gives runs with their attributes, which is where marks live.
for chunk in t.diff(txn, yrs::types::text::YChange::identity) {
let mut open = Vec::new();
if let Some(attrs) = &chunk.attributes {
- for (name, tag) in MARKS {
+ for (name, o, c) in MARKS {
if attrs.contains_key(*name) {
- open.push(*tag);
+ open.push((*o, *c));
}
}
}
- for tag in &open {
- let _ = write!(out, "<{tag}>");
+ for (o, _) in &open {
+ out.push_str(o);
}
if let yrs::Out::Any(yrs::Any::String(s)) = &chunk.insert {
escape(s, out);
}
- for tag in open.iter().rev() {
- let _ = write!(out, "</{tag}>");
+ for (_, c) in open.iter().rev() {
+ out.push_str(c);
}
}
}
@@ -193,3 +223,167 @@ pub fn fragment_text(doc: &Doc) -> String {
}
blocks.join("\n\n")
}
+
+// --- the structured renderer ---
+//
+// `fragment_html` above answers "what does this document look like";
+// `fragment_json` below answers "what is this document made of". The
+// export paths (`internal/export`, `internal/pdf`) need the second
+// question answered: where a list ends, which run is bold, which
+// paragraph is a heading. Re-deriving that in Go by parsing the markup
+// this file emits would mean a parser to keep honest against an
+// emitter, and Go's standard library has no HTML parser to do it with.
+//
+// The shape is ProseMirror's own JSON — `{"type":..,"attrs":..,"content":[..]}`,
+// text as `{"type":"text","text":..,"marks":[..]}` — because that is the
+// vocabulary the editor on the other side of the wire already speaks,
+// so a Go consumer and a future JS one describe the same document the
+// same way. Written by hand rather than through serde: the crate has
+// no serialisation dependency outside dev-dependencies, and this is
+// one escape function and three walks.
+
+/// JSON string escaping, per RFC 8259: the two mandatory escapes, the
+/// three shorthands, and `\u00XX` for everything else below U+0020.
+/// Everything at or above U+0020 other than `"` and `\` goes through
+/// as-is — the output is UTF-8 and JSON says that is legal.
+fn escape_json(s: &str, out: &mut String) {
+ for c in s.chars() {
+ match c {
+ '"' => out.push_str("\\\""),
+ '\\' => out.push_str("\\\\"),
+ '\n' => out.push_str("\\n"),
+ '\r' => out.push_str("\\r"),
+ '\t' => out.push_str("\\t"),
+ c if (c as u32) < 0x20 => {
+ let _ = write!(out, "\\u{:04x}", c as u32);
+ }
+ c => out.push(c),
+ }
+ }
+}
+
+/// The mark names on one text run: the ones this build knows in `MARKS`
+/// order, then any it does not, sorted.
+///
+/// The sort is not cosmetic. `Attrs` is a `HashMap`, so iterating it
+/// yields an order that varies between runs of the same binary on the
+/// same document — an unsorted tail would make this renderer's output
+/// nondeterministic, which would make every golden-file test on the Go
+/// side flake rather than fail. Unknown marks are carried through
+/// rather than dropped for the same reason unknown *nodes* render their
+/// children (see the module comment): a consumer that ignores a mark it
+/// does not recognise still has the text, and one that grows to
+/// understand it needs it to have been there.
+fn mark_names(attrs: &yrs::types::Attrs) -> Vec<String> {
+ let mut names: Vec<String> = MARKS
+ .iter()
+ .filter(|(name, _, _)| attrs.contains_key(*name))
+ .map(|(name, _, _)| (*name).to_string())
+ .collect();
+ let mut unknown: Vec<String> = attrs
+ .keys()
+ .map(|k| k.to_string())
+ .filter(|k| !MARKS.iter().any(|(name, _, _)| *name == k.as_str()))
+ .collect();
+ unknown.sort();
+ names.append(&mut unknown);
+ names
+}
+
+fn text_json<T: ReadTxn>(txn: &T, t: &XmlTextRef, out: &mut String, first: &mut bool) {
+ for chunk in t.diff(txn, yrs::types::text::YChange::identity) {
+ let yrs::Out::Any(yrs::Any::String(s)) = &chunk.insert else {
+ // A non-string insert is an embed. `fragment_html` renders
+ // nothing for one today, and inventing a JSON shape for
+ // something with no renderer would be guessing at a schema
+ // rather than reporting one.
+ continue;
+ };
+ if s.is_empty() {
+ continue;
+ }
+ if !*first {
+ out.push(',');
+ }
+ *first = false;
+ out.push_str("{\"type\":\"text\",\"text\":\"");
+ escape_json(s, out);
+ out.push('"');
+ if let Some(attrs) = &chunk.attributes {
+ let names = mark_names(attrs);
+ if !names.is_empty() {
+ out.push_str(",\"marks\":[");
+ for (i, name) in names.iter().enumerate() {
+ if i > 0 {
+ out.push(',');
+ }
+ out.push_str("{\"type\":\"");
+ escape_json(name, out);
+ out.push_str("\"}");
+ }
+ out.push(']');
+ }
+ }
+ out.push('}');
+ }
+}
+
+fn node_json<T: ReadTxn>(txn: &T, node: &XmlOut, out: &mut String, first: &mut bool) {
+ match node {
+ XmlOut::Text(t) => text_json(txn, t, out, first),
+ XmlOut::Element(e) => {
+ let name = e.tag().to_string();
+ if !*first {
+ out.push(',');
+ }
+ *first = false;
+ out.push_str("{\"type\":\"");
+ escape_json(&name, out);
+ out.push('"');
+ if name == "heading" {
+ // Clamped exactly as `node_html` clamps it, and for the
+ // same reason: a corrupt or hostile level must not reach
+ // a consumer as `h0` or `h99`. Both renderers clamping
+ // independently is deliberate — a document must not
+ // paginate as one level on screen and export as another.
+ let level = e
+ .get_attribute(txn, "level")
+ .and_then(|v| v.parse::<u8>().ok())
+ .unwrap_or(1)
+ .clamp(1, 6);
+ let _ = write!(out, ",\"attrs\":{{\"level\":{level}}}");
+ }
+ out.push_str(",\"content\":[");
+ let mut inner = true;
+ for child in e.children(txn) {
+ node_json(txn, &child, out, &mut inner);
+ }
+ out.push_str("]}");
+ }
+ XmlOut::Fragment(f) => {
+ // Flattened into the parent's content, exactly as
+ // `node_html` renders a nested fragment's children with no
+ // wrapper of its own.
+ for child in f.children(txn) {
+ node_json(txn, &child, out, first);
+ }
+ }
+ }
+}
+
+/// The document as ProseMirror-shaped JSON: `{"type":"doc","content":[..]}`.
+///
+/// Deterministic for a given document state — see `mark_names` for the
+/// one place that took work — so a Go consumer's golden files are
+/// stable.
+pub fn fragment_json(doc: &Doc) -> String {
+ let f = root(doc);
+ let txn = doc.transact();
+ let mut out = String::from("{\"type\":\"doc\",\"content\":[");
+ let mut first = true;
+ for child in f.children(&txn) {
+ node_json(&txn, &child, &mut out, &mut first);
+ }
+ out.push_str("]}");
+ out
+}
diff --git a/core/src/lib.rs b/core/src/lib.rs
index a132909..31e847a 100644
--- a/core/src/lib.rs
+++ b/core/src/lib.rs
@@ -55,7 +55,7 @@
//! it won't.
pub mod abi;
mod fragment;
-pub use fragment::{fragment_html, fragment_text, ROOT as FRAGMENT_ROOT};
+pub use fragment::{fragment_html, fragment_json, fragment_text, ROOT as FRAGMENT_ROOT};
#[cfg(target_arch = "wasm32")]
#[no_mangle]
@@ -71,10 +71,23 @@ extern "Rust" fn __getrandom_v03_custom(
/// box fails loudly instead of behaving strangely.
///
/// Bumped to 2 for the HTML read pair in `abi.rs`
-/// (`docs_core_html_len` / `docs_core_html_read`).
+/// (`docs_core_html_len` / `docs_core_html_read`), and to 3 for the JSON
+/// pair (`docs_core_json_len` / `docs_core_json_read`) the export paths
+/// read the document's structure through.
+///
+/// Adding an export is, strictly, backwards compatible — a host that
+/// never calls the new function cannot tell 3 from 2 — so the bump is
+/// not forced by the wasm itself. It is forced by the host: the module
+/// and the Go binary ship embedded in each other with no version skew
+/// between separately deployed copies, and `internal/core`'s
+/// `abiVersion` is an exact-match check. A host that calls
+/// `docs_core_json_len` against a stale module gets "unknown export" at
+/// call time, deep inside a request; bumping means it gets a refusal at
+/// load time instead, which is the whole point of there being a version
+/// here.
#[no_mangle]
pub extern "C" fn docs_core_abi_version() -> u32 {
- 2
+ 3
}
use yrs::updates::decoder::Decode;
diff --git a/core/tests/fragment.rs b/core/tests/fragment.rs
index 44e67c4..53100ac 100644
--- a/core/tests/fragment.rs
+++ b/core/tests/fragment.rs
@@ -1,6 +1,6 @@
-use docs_core::{fragment_html, fragment_text};
+use docs_core::{fragment_html, fragment_json, fragment_text};
use yrs::updates::decoder::Decode;
-use yrs::{Doc, Options, Transact, Update, XmlFragment};
+use yrs::{Doc, Options, Transact, Update, Xml, XmlFragment};
fn doc_from(bytes: &[u8]) -> Doc {
let doc = Doc::with_options(Options::with_guid_and_client_id("fixture".into(), 1));
@@ -144,3 +144,149 @@ fn nested_blocks_do_not_duplicate_their_text() {
}
assert_eq!(fragment_text(&doc), "x");
}
+
+// --- the font and size marks, and the structured renderer ---
+
+// The class names below are the contract between this renderer and
+// views/show.html's stylesheet, and between it and internal/pdf's
+// mark handling. They are constants in this binary, never bytes from a
+// document — see the MARKS table's own comment.
+#[test]
+fn renders_font_and_size_marks_as_classed_spans() {
+ let doc = doc_from(include_bytes!("../../fixtures/vectors/fragment/styled.bin"));
+ assert_eq!(
+ fragment_html(&doc),
+ concat!(
+ "<p>",
+ "<span class=\"docs-font-sans\">sans</span>",
+ "<span class=\"docs-font-mono\"> mono</span>",
+ "<span class=\"docs-size-lg\"> large</span>",
+ "<span class=\"docs-font-sans\"><strong> both</strong></span>",
+ "</p>",
+ "<blockquote><p>Quoted.</p></blockquote>",
+ "<ol><li><p>First</p></li><li><p>Second</p></li></ol>",
+ )
+ );
+}
+
+#[test]
+fn renders_the_document_as_prosemirror_json() {
+ let doc = doc_from(include_bytes!("../../fixtures/vectors/fragment/nested.bin"));
+ assert_eq!(
+ fragment_json(&doc),
+ concat!(
+ "{\"type\":\"doc\",\"content\":[",
+ "{\"type\":\"heading\",\"attrs\":{\"level\":2},\"content\":[",
+ "{\"type\":\"text\",\"text\":\"A heading\"}]},",
+ "{\"type\":\"bullet_list\",\"content\":[",
+ "{\"type\":\"list_item\",\"content\":[{\"type\":\"paragraph\",\"content\":[",
+ "{\"type\":\"text\",\"text\":\"One\"}]}]},",
+ "{\"type\":\"list_item\",\"content\":[{\"type\":\"paragraph\",\"content\":[",
+ "{\"type\":\"text\",\"text\":\"Two\"}]}]}]}]}",
+ )
+ );
+}
+
+// Marks come out in MARKS order, not in whatever order the CRDT's own
+// attribute map happens to iterate — see mark_names. A run carrying two
+// of them is what actually exercises that.
+#[test]
+fn json_marks_are_in_a_fixed_order() {
+ let doc = doc_from(include_bytes!("../../fixtures/vectors/fragment/styled.bin"));
+ let json = fragment_json(&doc);
+ assert!(
+ json.contains(concat!(
+ "{\"type\":\"text\",\"text\":\" both\",\"marks\":[",
+ "{\"type\":\"font_sans\"},{\"type\":\"strong\"}]}",
+ )),
+ "got {json}"
+ );
+}
+
+// Same document, rendered twice in one process: identical bytes. A
+// HashMap-ordered mark list would pass this only by luck, which is why
+// the run above carries two marks.
+#[test]
+fn json_is_deterministic() {
+ let doc = doc_from(include_bytes!("../../fixtures/vectors/fragment/styled.bin"));
+ assert_eq!(fragment_json(&doc), fragment_json(&doc));
+}
+
+// The JSON goes into an HTTP response body and, from there, into a Go
+// json.Unmarshal. A document whose text contains quotes, backslashes or
+// control characters must not be able to break out of its string.
+#[test]
+fn escapes_json_in_text() {
+ let doc = Doc::with_options(Options::with_guid_and_client_id("jesc".into(), 1));
+ {
+ let f = doc.get_or_insert_xml_fragment("prosemirror");
+ let mut txn = doc.transact_mut();
+ let p = f.push_back(&mut txn, yrs::XmlElementPrelim::empty("paragraph"));
+ p.push_back(
+ &mut txn,
+ yrs::XmlTextPrelim::new("a \"quote\" and a \\ and a \u{1}"),
+ );
+ }
+ assert_eq!(
+ fragment_json(&doc),
+ concat!(
+ "{\"type\":\"doc\",\"content\":[{\"type\":\"paragraph\",\"content\":[",
+ "{\"type\":\"text\",\"text\":\"a \\\"quote\\\" and a \\\\ and a \\u0001\"}]}]}",
+ )
+ );
+}
+
+// The module's rule — unknown nodes render their children rather than
+// disappearing — holds for the structured renderer too, with the node
+// itself reported under its own name so a consumer that grows to
+// understand it can. A newer editor build introducing a node this
+// binary has never heard of must not lose its text on export.
+#[test]
+fn json_keeps_unknown_nodes_and_their_children() {
+ let doc = Doc::with_options(Options::with_guid_and_client_id("junk".into(), 1));
+ {
+ let f = doc.get_or_insert_xml_fragment("prosemirror");
+ let mut txn = doc.transact_mut();
+ let outer = f.push_back(&mut txn, yrs::XmlElementPrelim::empty("table"));
+ let p = outer.push_back(&mut txn, yrs::XmlElementPrelim::empty("paragraph"));
+ p.push_back(&mut txn, yrs::XmlTextPrelim::new("kept"));
+ }
+ assert_eq!(
+ fragment_json(&doc),
+ concat!(
+ "{\"type\":\"doc\",\"content\":[{\"type\":\"table\",\"content\":[",
+ "{\"type\":\"paragraph\",\"content\":[{\"type\":\"text\",\"text\":\"kept\"}]}]}]}",
+ )
+ );
+}
+
+// A heading level is clamped identically by both renderers. A document
+// that reads as an h6 on screen must not export as an h99 — the two
+// clamps are written separately, so this is the test that keeps them
+// equal.
+#[test]
+fn json_clamps_heading_level_like_html() {
+ let doc = Doc::with_options(Options::with_guid_and_client_id("hclamp".into(), 1));
+ {
+ let f = doc.get_or_insert_xml_fragment("prosemirror");
+ let mut txn = doc.transact_mut();
+ let h = f.push_back(&mut txn, yrs::XmlElementPrelim::empty("heading"));
+ h.insert_attribute(&mut txn, "level", "99");
+ h.push_back(&mut txn, yrs::XmlTextPrelim::new("Shouty"));
+ }
+ assert!(fragment_html(&doc).starts_with("<h6>"), "html clamp");
+ assert!(
+ fragment_json(&doc).contains("\"attrs\":{\"level\":6}"),
+ "json clamp: {}",
+ fragment_json(&doc)
+ );
+}
+
+// An empty document is still a document: `{"type":"doc","content":[]}`,
+// never the empty string. internal/core's json read relies on that —
+// see abi.rs's json_string.
+#[test]
+fn json_of_an_empty_document_is_an_empty_doc_node() {
+ let doc = Doc::with_options(Options::with_guid_and_client_id("empty".into(), 1));
+ assert_eq!(fragment_json(&doc), "{\"type\":\"doc\",\"content\":[]}");
+}
diff --git a/docs/superpowers/plans/2026-08-31-formatting-and-export.md b/docs/superpowers/plans/2026-08-31-formatting-and-export.md
new file mode 100644
index 0000000..4c07e76
--- /dev/null
+++ b/docs/superpowers/plans/2026-08-31-formatting-and-export.md
@@ -0,0 +1,208 @@
+# Plan 5 — the formatting bar, font options, and export
+
+Brief, from Paul, 2026-08-31: *"the maximally viable checklist — a
+formatting bar, some font options, at least PDF and Markdown export."*
+
+The spec already rules on all four (§11 Fonts/DOCX/PDF/Markdown, §13's
+sequencing item 7: "Markdown, then DOCX, then PDF"). This plan is that
+item, brought forward past DOCX — DOCX is import *and* export and needs
+the schema work §11.2 describes; Markdown and PDF need only what the
+CRDT already holds. Nothing here changes a spec ruling. Two places
+where this plan reads a ruling differently from a literal reading of
+§11 are recorded as D1 and D2 below, for the branch discussion to
+settle rather than for this plan to settle by itself.
+
+## What exists to build on
+
+- The document's truth is a `Y.XmlFragment` on root `prosemirror`
+ (`core/src/fragment.rs`), replayed server-side through wazero.
+ `fragment_html` and `fragment_text` are its two renderers today.
+- `internal/core` exposes that through a C ABI (`docs_core_html_len` /
+ `_read`, and the text pair). `abiVersion` is 2.
+- The browser editor is ProseMirror + Yjs, one committed bundle
+ (`fixtures/build-editor.mjs` → `internal/docs/static/editor.<hash>.js`).
+ `prosemirror-schema-basic` + `addListNodes` is the schema; `Mod-b` and
+ `Mod-i` are already bound to `toggleMark`, so bold and italic exist
+ today with no way to *see* that they exist.
+- Every user-visible string is a catalog key (`internal/docs/render.go`).
+
+## The shape of the work
+
+Seven tasks. 1–3 are server-side and touch none of the files the
+concurrent caret-placement branch touches; 5–6 touch the editor bundle
+and are deliberately sequenced after that branch merges.
+
+### Task 1 — the core renders a structured document, not only markup
+
+`fragment.rs` gains `fragment_json`, a third renderer over the same
+walk: ProseMirror's own JSON shape (`{"type":…,"attrs":…,"content":[…]}`,
+text nodes as `{"type":"text","text":…,"marks":[…]}`), hand-written with
+no serde dependency (the crate has none outside dev-dependencies, and
+this is forty lines of escaping).
+
+The ABI gains `docs_core_json_len` / `docs_core_json_read`, the same
+len-then-read pair the other two use, and `abiVersion` goes to 3.
+
+**Why a third renderer rather than parsing the HTML in Go.** Markdown
+and PDF both need structure — where a list ends, which run is bold,
+which paragraph is a heading. Go's standard library has no HTML parser,
+and `golang.org/x/net/html` is a new module dependency this repo does
+not have and cannot fetch offline. Re-deriving structure from our own
+markup would also mean two parsers to keep honest against one emitter.
+One structured export, two consumers in Go, is fewer moving parts than
+either alternative.
+
+**Unknown nodes still render their children** — `fragment.rs`'s module
+rule, and it holds here for the same reason: a reader (or an exporter)
+silently dropping text is data loss, not a degradation.
+
+### Task 2 — font and size as marks the core knows
+
+Five new marks, all attribute-free: `font_sans`, `font_mono` (serif is
+the document default and carries no mark), and `size_sm`, `size_lg`,
+`size_xl` (normal carries no mark).
+
+`fragment.rs`'s `MARKS` table gains them, emitting a `<span>` with a
+fixed class — `<span class="docs-font-sans">` — chosen by lookup from a
+compile-time table.
+
+**Attribute-free marks, not a `fontFamily` mark with an attribute.**
+`Store.HTML` returns `template.HTML` — markup trusted to be written out
+unescaped — and its doc comment says exactly what earns that trust:
+`fragment.rs` escapes every text run *and emits no attributes at all*.
+A mark carrying a value from the CRDT into attribute position would put
+client-supplied bytes inside a quoted attribute, which is a different
+escaping problem with a different set of ways to get it wrong (and the
+CRDT accepts arbitrary client-posted bytes — `POST /d/{sqid}/updates`
+does not go through ProseMirror's schema). A fixed set of marks whose
+class names are constants in this binary keeps the property that made
+the `template.HTML` return safe in the first place: no byte from a
+document ever reaches attribute position.
+
+The cost is that the set is closed — a new size is a core change and a
+new binary, not a document that names its own point size. For "some
+font options" that is the right trade; §14 is where an open-ended font
+model goes if it is ever wanted.
+
+### Task 3 — `internal/fonts`: the core set, embedded
+
+Liberation Serif, Sans and Mono, four faces each (regular, bold,
+italic, bold-italic), embedded with `//go:embed` and served at
+`/static/fonts/…` behind the same immutable caching the editor bundle
+uses.
+
+**D1 — Liberation, where §11.1 names Carlito and Caladea.** §11.1 asks
+for two things that are separable, and this plan does the first
+without the second. The first is *a curated core set, self-hosted, the
+same font programs on the screen and in server-side layout, so both
+measure identically* — that is what makes a PDF match what the writer
+saw, and it is met here. The second is *metric-compatible substitution
+for the proprietary faces a DOCX import names*: Calibri → Carlito,
+Cambria → Caladea. That mapping only has a job once DOCX import exists
+to name Calibri, and there is no DOCX import. Liberation is the same
+idea one family over (it is the metric-compatible set for Times New
+Roman, Arial and Courier New), it is OFL-licensed, and it is what is
+actually available to this build. When DOCX import lands, Carlito and
+Caladea join the payload; nothing here forecloses that.
+
+**woff2 is not in this plan.** §11.1 says woff2 and means it — the TTFs
+are roughly twice the bytes. Converting requires `woff2_compress` or
+fontTools, neither of which is available here, and hand-writing a
+Brotli-based woff2 encoder to save download bytes is not a good use of
+this branch. The faces are declared with `@font-face` and a browser
+fetches only the faces a page actually uses, so an unused face costs
+nothing; the conversion is a follow-up with a real trigger (the first
+time page weight is measured and this shows up in it).
+
+### Task 4 — `internal/pdf`: a real PDF writer
+
+Pure Go, no new module dependencies, `CGO_ENABLED=0` like everything
+else:
+
+- **A TrueType reader** (`head`, `hhea`, `maxp`, `hmtx`, `cmap` format
+ 4, `loca`, `glyf`) for advance widths and glyph outlines.
+- **A subsetter**: the glyphs a document actually uses, with `glyf`,
+ `loca` and `hmtx` rebuilt and composite glyphs' components followed
+ so an accented character does not lose its accent.
+- **The writer**: objects, an xref table, Flate-compressed content
+ streams, and a Type0/CIDFontType2 font with Identity-H encoding plus
+ a `ToUnicode` CMap, so text in the exported PDF is selectable and
+ searchable rather than a picture of text.
+- **Layout** from the same page geometry the screen uses: US Letter,
+ 1in margins, 12pt/1.6 body — the constants live in Go and
+ `internal/docs` has a test that fails if `views/show.html` stops
+ agreeing with them. §11.3's "the same box tree the screen renders"
+ is a claim about the *engine* the layout plan will build; until that
+ exists this is the honest version — one set of constants, asserted
+ equal in both places, and a note that they are asserted rather than
+ shared.
+
+**D2 — this is not the layout engine.** §13 says the layout engine
+cannot slip because pagination, PDF and thumbnails all fall out of it,
+and the engine is not built. What Task 4 builds is a line-breaker and a
+paginator for the document shapes that exist today (paragraphs,
+headings, lists, blockquotes, inline marks) — real text measurement
+from real font metrics, not an approximation, but it knows nothing
+about tables, images, floats or widow/orphan control. When the layout
+engine lands, this becomes its back end (the writer, the subsetter and
+the font work all survive) and the line-breaking half of it is
+replaced. Recording it here so a later reader does not mistake this for
+the engine and build the engine's other consumers on top of it.
+
+### Task 5 — the formatting bar
+
+`fixtures/toolbar.mjs`, a new module under the 300-line cap, wired into
+`editor-entry.js` in a few lines. Bold, italic, code, link; paragraph
+and headings 1–3; bullet and ordered lists; blockquote; undo and redo;
+a font-family control and a size control driving Task 2's marks.
+
+Chrome, so it lives outside `[docs-editor]` — ProseMirror's
+`{mount: el}` takes over everything inside that element. Buttons
+reflect the selection's active marks, targets are 24×24 (§10.3, WCAG
+2.2 AA, no exemptions), every label is a catalog key rendered
+server-side into the markup the way the save-status copy already is —
+the bundle carries no copy of its own.
+
+Each command dispatches one transaction, so one toolbar click costs one
+retained update, not several: `internal/core`'s `maxRetainedUpdates` is
+4096 with no compaction yet, and toolbar-driven editing spends it
+faster than typing does.
+
+### Task 6 — the export routes and where a person clicks them
+
+`GET /d/{sqid}/export.md` and `GET /d/{sqid}/export.pdf`, both inside
+the `RequireSession` group and both owner-scoped through the same
+`Store.Get` every other document route uses — a document someone else
+owns 404s identically to one that does not exist (§4.2), with no
+separate check that could 403 by mistake.
+
+`Content-Disposition: attachment` with a filename derived from the
+document title, sanitised to a conservative set and falling back to the
+sqid — a title is arbitrary user text and a header is the wrong place
+to find that out.
+
+The controls go in `show.html`'s page header next to sign-out.
+
+### Task 7 — the gate
+
+`.amadan/ci.d/10-gate` already covers everything this adds (Rust tests,
+Go tests, the bundle-freshness check, gofmt). What this task adds is
+the tests themselves: fragment JSON fixtures on the Rust side, Markdown
+golden files, a PDF that is parsed back and asserted structurally
+(pages, fonts embedded, text extractable) rather than byte-compared,
+the geometry agreement test, and a toolbar test under `node --test` for
+whatever logic can be tested without a DOM.
+
+## Risks
+
+- **The bundle is a single committed artifact and only one may exist**
+ (`assets.go` panics at init otherwise). This branch rebuilds it once,
+ at the end, after the concurrent caret-placement branch has merged —
+ a rebuild is not hand-mergeable.
+- **`maxRetainedUpdates` = 4096, no compaction.** Not this branch's to
+ fix, made worse by this branch. Task 5's one-transaction-per-command
+ rule is the mitigation available without opening compaction.
+- **PDF is the long pole.** If it does not finish, Markdown, the
+ toolbar and the fonts still ship on their own and the branch is
+ still worth merging; the reverse is not true, so PDF goes last of
+ the server-side tasks and first of the ones that could be cut.
diff --git a/fixtures/generate-fragment.mjs b/fixtures/generate-fragment.mjs
index 1f1b332..1a213e8 100644
--- a/fixtures/generate-fragment.mjs
+++ b/fixtures/generate-fragment.mjs
@@ -54,6 +54,27 @@ writeFileSync('vectors/fragment/nested.bin', docWith(f => {
])
}))
+// styled.bin covers what the formatting bar can produce that the three
+// fixtures above cannot: the font and size marks (core/src/fragment.rs's
+// MARKS table), a run carrying two marks at once, and the two block
+// shapes — blockquote and ordered_list — that had no fixture before the
+// export paths needed to render them. The marks are attribute-free on
+// purpose; see that table's own comment for why a mark never carries a
+// value.
+writeFileSync('vectors/fragment/styled.bin', docWith(f => {
+ f.insert(0, [
+ el('paragraph', null,
+ text('sans', { font_sans: {} }),
+ text(' mono', { font_mono: {} }),
+ text(' large', { size_lg: {} }),
+ text(' both', { font_sans: {}, strong: {} })),
+ el('blockquote', null, el('paragraph', null, text('Quoted.'))),
+ el('ordered_list', null,
+ el('list_item', null, el('paragraph', null, text('First'))),
+ el('list_item', null, el('paragraph', null, text('Second')))),
+ ])
+}))
+
// --- trap-containment vectors ---
//
// core/tests/wasm_trap_containment.rs mutation-fuzzes real fixtures and
@@ -168,4 +189,4 @@ emitTrap('concurrent', (doc) => {
Y.applyUpdate(doc, Y.encodeStateAsUpdate(a))
})
-console.log('wrote 3 fragment vectors + 4 trap-containment vectors')
+console.log('wrote 4 fragment vectors + 4 trap-containment vectors')
diff --git a/fixtures/vectors/fragment/styled.bin b/fixtures/vectors/fragment/styled.bin
new file mode 100644
index 0000000..df87016
Binary files /dev/null and b/fixtures/vectors/fragment/styled.bin differ
diff --git a/internal/core/core.go b/internal/core/core.go
index da6b0ac..188fc04 100644
--- a/internal/core/core.go
+++ b/internal/core/core.go
@@ -62,7 +62,7 @@ var moduleWASM []byte
// anything else is refused at load, so a stale .wasm (see
// scripts/build-wasm.sh and the copy step in Task 6's brief) fails loudly
// rather than behaving strangely.
-const abiVersion = 2
+const abiVersion = 3
// defaultGuestCallTimeout bounds every single call into the wasm module,
// unless overridden with WithGuestCallTimeout.
@@ -529,78 +529,45 @@ func (h *Handle) Apply(ctx context.Context, update []byte) error {
// empty string plus an error instead of it would tell the caller the data
// is gone when it is not.
func (h *Handle) Text(ctx context.Context) (string, error) {
- text, err := textOn(ctx, h.rt.callTimeout, h.mod, h.handle)
- if err == nil {
- return text, nil
- }
- if !isFatal(err) {
- return "", err
- }
- if rebuildErr := h.rebuild(ctx); rebuildErr != nil {
- return "", errors.Join(err, rebuildErr)
- }
- text, err = textOn(ctx, h.rt.callTimeout, h.mod, h.handle)
- if err != nil {
- return "", fmt.Errorf("reading text from the rebuilt instance: %w", err)
- }
- return text, nil
+ return h.read(ctx, "text")
}
-func textOn(ctx context.Context, timeout time.Duration, mod api.Module, docHandle uint64) (string, error) {
- lenRes, err := callGuest(ctx, timeout, mod.ExportedFunction("docs_core_text_len"), docHandle)
- if err != nil {
- return "", fmt.Errorf("text_len trapped or timed out: %w", err)
- }
- // text_len returns i32: negative is an error code, not a length. Reading
- // it as unsigned would turn ERR_NO_DOC into 0xFFFFFFFF and then try to
- // allocate four gigabytes.
- if code := int32(lenRes[0]); code < 0 {
- return "", fmt.Errorf("text_len failed: code %d", code)
- }
- n := lenRes[0]
- if n == 0 {
- return "", nil
- }
-
- ptr, err := allocIn(ctx, timeout, mod, n)
- if err != nil {
- return "", fmt.Errorf("alloc: %w", err)
- }
- defer freeIn(ctx, timeout, mod, ptr, n)
-
- rc, err := callGuest(ctx, timeout, mod.ExportedFunction("docs_core_text_read"), docHandle, ptr, n)
- if err != nil {
- return "", fmt.Errorf("text_read trapped or timed out: %w", err)
- }
- written := int32(rc[0])
- if written < 0 {
- return "", fmt.Errorf("text_read failed: code %d", written)
- }
+// HTML returns the document's current content rendered as HTML (spec
+// §6.2: the read view is server-rendered, no script required).
+func (h *Handle) HTML(ctx context.Context) (string, error) {
+ return h.read(ctx, "html")
+}
- // docs_core_text_read returns s.len(), guaranteed only <= out_len
- // (core/src/abi.rs:287 — the buffer sized by text_len is an upper
- // bound, not a promise the two calls agree exactly). Reading n bytes
- // instead of the written count would, if they ever diverged, append
- // uninitialised allocator bytes past the real content to the returned
- // string (fix-round-1 finding 3) — read exactly what text_read says it
- // wrote.
- buf, ok := mod.Memory().Read(uint32(ptr), uint32(written))
- if !ok {
- return "", fmt.Errorf("reading %d bytes at %d: out of range", written, ptr)
- }
- return string(buf), nil
+// JSON returns the document's structure as ProseMirror-shaped JSON —
+// what the export paths (internal/export, internal/pdf) read, since
+// Markdown and PDF both need to know where a list ends and which run is
+// bold, and neither can get that from HTML without a parser. See
+// core/src/fragment.rs's fragment_json.
+//
+// The bytes are always a complete JSON document for a live handle,
+// never "" — an empty document is {"type":"doc","content":[]} — so a
+// caller may unmarshal the result without a special case for empty.
+func (h *Handle) JSON(ctx context.Context) (string, error) {
+ return h.read(ctx, "json")
}
-// HTML returns the document's current content rendered as HTML (spec
-// §6.2: the read view is server-rendered, no script required). Mirrors
-// Text exactly, including the trap/timeout-then-rebuild-then-retry-once
-// behaviour: if the read itself traps or times out, the instance is
-// discarded and rebuilt, and the read is retried once against the freshly
-// rebuilt instance rather than reporting the content as gone.
-func (h *Handle) HTML(ctx context.Context) (string, error) {
- html, err := htmlOn(ctx, h.rt.callTimeout, h.mod, h.handle)
+// read is the shared body of Text, HTML and JSON: call the named
+// renderer's len/read pair, and if the call itself traps or times out,
+// discard the instance, rebuild it exactly as Apply does, and retry the
+// read once against the fresh one. Retrying rather than reporting an
+// error is deliberate — the content is available at that point, and
+// returning an empty string plus an error would tell the caller the
+// data is gone when it is not.
+//
+// One function rather than three near-identical ones. It was three
+// (textOn/htmlOn, and jsonOn was about to be the third): the same forty
+// lines with two identifiers substituted, which is three places for a
+// fix like fix-round-1's finding 3 (read the written count, not the
+// buffer size) to be applied to two of.
+func (h *Handle) read(ctx context.Context, renderer string) (string, error) {
+ s, err := readOn(ctx, h.rt.callTimeout, h.mod, h.handle, renderer)
if err == nil {
- return html, nil
+ return s, nil
}
if !isFatal(err) {
return "", err
@@ -608,26 +575,40 @@ func (h *Handle) HTML(ctx context.Context) (string, error) {
if rebuildErr := h.rebuild(ctx); rebuildErr != nil {
return "", errors.Join(err, rebuildErr)
}
- html, err = htmlOn(ctx, h.rt.callTimeout, h.mod, h.handle)
+ s, err = readOn(ctx, h.rt.callTimeout, h.mod, h.handle, renderer)
if err != nil {
- return "", fmt.Errorf("reading html from the rebuilt instance: %w", err)
+ return "", fmt.Errorf("reading %s from the rebuilt instance: %w", renderer, err)
}
- return html, nil
+ return s, nil
}
-// htmlOn is textOn's HTML counterpart — the same two-call
-// len-then-read dance against docs_core_html_len/docs_core_html_read
-// instead of the text pair.
-func htmlOn(ctx context.Context, timeout time.Duration, mod api.Module, docHandle uint64) (string, error) {
- lenRes, err := callGuest(ctx, timeout, mod.ExportedFunction("docs_core_html_len"), docHandle)
+// readOn performs one renderer's len-then-read dance against
+// docs_core_<renderer>_len and docs_core_<renderer>_read. renderer is
+// always one of this package's own three literals ("text", "html",
+// "json"), never anything derived from a request — the export names it
+// builds are a closed set, and a name that does not resolve to an
+// exported function surfaces below as a nil api.Function.
+func readOn(ctx context.Context, timeout time.Duration, mod api.Module, docHandle uint64, renderer string) (string, error) {
+ lenFn := mod.ExportedFunction("docs_core_" + renderer + "_len")
+ readFn := mod.ExportedFunction("docs_core_" + renderer + "_read")
+ if lenFn == nil || readFn == nil {
+ // Unreachable while abiVersion matches — the version check at
+ // load is what proves these exports exist — but a nil
+ // api.Function panics inside callGuest rather than erroring, so
+ // this turns a future ABI mistake into a returned error instead
+ // of a panic in a request handler.
+ return "", fmt.Errorf("module exports no %q renderer", renderer)
+ }
+
+ lenRes, err := callGuest(ctx, timeout, lenFn, docHandle)
if err != nil {
- return "", fmt.Errorf("html_len trapped or timed out: %w", err)
+ return "", fmt.Errorf("%s_len trapped or timed out: %w", renderer, err)
}
- // html_len returns i32: negative is an error code, not a length. Reading
- // it as unsigned would turn ERR_NO_DOC into 0xFFFFFFFF and then try to
- // allocate four gigabytes.
+ // The len functions return i32: negative is an error code, not a
+ // length. Reading it as unsigned would turn ERR_NO_DOC into
+ // 0xFFFFFFFF and then try to allocate four gigabytes.
if code := int32(lenRes[0]); code < 0 {
- return "", fmt.Errorf("html_len failed: code %d", code)
+ return "", fmt.Errorf("%s_len failed: code %d", renderer, code)
}
n := lenRes[0]
if n == 0 {
@@ -640,21 +621,22 @@ func htmlOn(ctx context.Context, timeout time.Duration, mod api.Module, docHandl
}
defer freeIn(ctx, timeout, mod, ptr, n)
- rc, err := callGuest(ctx, timeout, mod.ExportedFunction("docs_core_html_read"), docHandle, ptr, n)
+ rc, err := callGuest(ctx, timeout, readFn, docHandle, ptr, n)
if err != nil {
- return "", fmt.Errorf("html_read trapped or timed out: %w", err)
+ return "", fmt.Errorf("%s_read trapped or timed out: %w", renderer, err)
}
written := int32(rc[0])
if written < 0 {
- return "", fmt.Errorf("html_read failed: code %d", written)
+ return "", fmt.Errorf("%s_read failed: code %d", renderer, written)
}
- // docs_core_html_read returns s.len(), guaranteed only <= out_len — the
- // buffer sized by html_len is an upper bound, not a promise the two
- // calls agree exactly, exactly as docs_core_text_read. Reading n bytes
+ // The read functions return s.len(), guaranteed only <= out_len
+ // (core/src/abi.rs — the buffer sized by the len call is an upper
+ // bound, not a promise the two calls agree exactly). Reading n bytes
// instead of the written count would, if they ever diverged, append
- // uninitialised allocator bytes past the real content to the returned
- // string — read exactly what html_read says it wrote.
+ // uninitialised allocator bytes past the real content to the
+ // returned string (fix-round-1 finding 3) — read exactly what the
+ // read call says it wrote.
buf, ok := mod.Memory().Read(uint32(ptr), uint32(written))
if !ok {
return "", fmt.Errorf("reading %d bytes at %d: out of range", written, ptr)
diff --git a/internal/core/testdata/docs_core.wasm b/internal/core/testdata/docs_core.wasm
index 6a93072..45e7c36 100755
Binary files a/internal/core/testdata/docs_core.wasm and b/internal/core/testdata/docs_core.wasm differ