oficina / docs Public

Core encodes a whole document as one update (ABI v4)

Adds snapshot_bytes(doc: &Doc) -> Vec<u8>, encoding the full document
state via encode_state_as_update_v1 against an empty StateVector - the
merge-equivalent (not merely render-equivalent) encoding that preserves
every clock range, tombstone, and pending/parked update, unlike the
neighbouring encode_diff_v1 which silently drops the pending queue.

Exports it through the wasm ABI as docs_core_snapshot_len /
docs_core_snapshot_read, matching the existing text/html/json
len-then-read convention, and bumps docs_core_abi_version to 4 for the
added export pair.

Bumps internal/core's abiVersion to 4 to match - the Go host's version
check is exact-match, so the two sides of this contract must move
together or the host refuses to load the module. Also exports
core.ABIVersion and core.MaxRetainedUpdates as aliases of the existing
unexported constants (no duplicated numbers) for later snapshot-storage
work to consume.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>

Paul Campbell pushed by paul@keymail.dev 1 month ago 442fb91ccad014ba3368d90d1c895c94ad6e8646 parent 62dbede

6 files changed, +155 −21

core/src/abi.rs +41 −1
core/src/lib.rs +7 −3
core/src/snapshot.rs +32 −0
core/tests/snapshot.rs +34 −0
internal/core/core.go +41 −17
internal/core/testdata/docs_core.wasm binary

diff --git a/core/src/abi.rs b/core/src/abi.rs

index 9901174..277b4fc 100644

--- a/core/src/abi.rs

+++ b/core/src/abi.rs

@@ -46,7 +46,7 @@

//! held a stale handle is exactly the kind of silent corruption §5.2

//! exists to avoid.

-use crate::{apply_update, fragment_html, fragment_json, fragment_text, CoreError};

+use crate::{apply_update, fragment_html, fragment_json, fragment_text, snapshot_bytes, CoreError};

use std::cell::RefCell;

use std::collections::HashMap;

use yrs::{Doc, Options};

@@ -159,6 +159,13 @@ pub fn json_string(handle: u32) -> Option<String> {

get_doc(handle).map(|doc| fragment_json(&doc))

}

+/// The whole document as one update. Bytes, not a string — the host

+/// reads it through the same len/read convention as the three

+/// renderers, which is why this is shaped like them.

+pub fn snapshot_vec(handle: u32) -> Option<Vec<u8>> {

+ get_doc(handle).map(|doc| snapshot_bytes(&doc))

// --- wasm exports ---

// Amended by pre-flight ruling R2. The obvious version — Vec::with_capacity

@@ -407,3 +414,36 @@ pub unsafe extern "C" fn docs_core_json_read(handle: u32, out_ptr: u32, out_len:

// Same clamp, same reason as the two read functions above.

s.len().try_into().unwrap_or(i32::MAX)

}

+/// Returns the byte length of `handle`'s whole-state snapshot, or

+/// `ERR_NO_DOC` if `handle` doesn't name a live document — distinct

+/// from 0, exactly as `docs_core_text_len`, `docs_core_html_len`, and

+/// `docs_core_json_len` above.

+#[no_mangle]

+pub extern "C" fn docs_core_snapshot_len(handle: u32) -> i32 {

+ match snapshot_vec(handle) {

+ Some(v) => v.len().try_into().unwrap_or(i32::MAX),

+ None => ERR_NO_DOC,

+ }

+/// # Safety

+///

+/// `out_ptr` and `out_len` must describe a byte range that lies entirely

+/// within this module's linear memory and is valid for writes for the

+/// duration of this call.

+#[no_mangle]

+pub unsafe extern "C" fn docs_core_snapshot_read(handle: u32, out_ptr: u32, out_len: u32) -> i32 {

+ let v = match snapshot_vec(handle) {

+ Some(v) => v,

+ None => return ERR_NO_DOC,

+ };

+ if v.len() > out_len as usize {

+ return ERR_BUFFER_TOO_SMALL;

+ }

+ if !in_bounds(out_ptr, out_len) {

+ return ERR_INVALID_LENGTH;

+ }

+ std::ptr::copy_nonoverlapping(v.as_ptr(), out_ptr as *mut u8, v.len());

+ v.len().try_into().unwrap_or(i32::MAX)

diff --git a/core/src/lib.rs b/core/src/lib.rs

index 31e847a..c425d91 100644

--- a/core/src/lib.rs

+++ b/core/src/lib.rs

@@ -55,7 +55,9 @@

//! it won't.

pub mod abi;

mod fragment;

+mod snapshot;

pub use fragment::{fragment_html, fragment_json, fragment_text, ROOT as FRAGMENT_ROOT};

+pub use snapshot::snapshot_bytes;

#[cfg(target_arch = "wasm32")]

#[no_mangle]

@@ -71,9 +73,11 @@ extern "Rust" fn __getrandom_v03_custom(

/// box fails loudly instead of behaving strangely.

///

/// Bumped to 2 for the HTML read pair in `abi.rs`

-/// (`docs_core_html_len` / `docs_core_html_read`), and to 3 for the JSON

+/// (`docs_core_html_len` / `docs_core_html_read`), to 3 for the JSON

/// pair (`docs_core_json_len` / `docs_core_json_read`) the export paths

-/// read the document's structure through.

+/// read the document's structure through, and to 4 for the snapshot

+/// pair (`docs_core_snapshot_len` / `docs_core_snapshot_read`) that

+/// exports a document's whole state as one update.

///

/// Adding an export is, strictly, backwards compatible — a host that

/// never calls the new function cannot tell 3 from 2 — so the bump is

@@ -87,7 +91,7 @@ extern "Rust" fn __getrandom_v03_custom(

/// here.

#[no_mangle]

pub extern "C" fn docs_core_abi_version() -> u32 {

- 3

+ 4

}

use yrs::updates::decoder::Decode;

diff --git a/core/src/snapshot.rs b/core/src/snapshot.rs

new file mode 100644

index 0000000..5694341

--- /dev/null

+++ b/core/src/snapshot.rs

@@ -0,0 +1,32 @@

+//! Encoding a whole document as one update, and checking that the

+//! result is the document it claims to be.

+//!

+//! The call below is `encode_state_as_update_v1` against an EMPTY state

+//! vector, and which call this is matters more than anything else in

+//! this file. Against the empty vector it encodes every block, the

+//! entire delete set, and — through yrs's own `merge_pending_v1` — the

+//! pending queue, meaning updates whose dependencies never arrived and

+//! which `apply_update` accepted and parked. That makes the result

+//! merge-equivalent to the log it replaces, not merely

+//! render-equivalent: every clock range and every tombstone survives,

+//! so a later concurrent merge integrates against a snapshot exactly as

+//! it would against the full history.

+//!

+//! Two neighbours are traps, and both look like this one:

+//!

+//! - `encode_diff_v1(&StateVector::default())` silently drops the

+//! pending queue (its own doc comment says so). A document holding an

+//! update whose dependency arrives later heals under a full replay and

+//! does NOT heal under a diff-built snapshot — and no render

+//! comparison can see the difference, because pending content is

+//! invisible in both documents.

+//! - yrs's own `Snapshot` type and `encode_state_from_snapshot` are a

+//! different feature (historical views, requiring `skip_gc`) that

+//! happens to share this module's name.

+use yrs::{Doc, ReadTxn, StateVector, Transact};

+/// The whole document, as one update.

+pub fn snapshot_bytes(doc: &Doc) -> Vec<u8> {

+ doc.transact()

+ .encode_state_as_update_v1(&StateVector::default())

diff --git a/core/tests/snapshot.rs b/core/tests/snapshot.rs

new file mode 100644

index 0000000..96a0300

--- /dev/null

+++ b/core/tests/snapshot.rs

@@ -0,0 +1,34 @@

+use docs_core::{fragment_text, snapshot_bytes};

+use yrs::updates::decoder::Decode;

+use yrs::{Doc, Options, Transact};

+fn doc_with(client_id: u64) -> Doc {

+ Doc::with_options(Options::with_guid_and_client_id(

+ "test-guid".into(),

+ client_id,

+ ))

+#[test]

+fn a_snapshot_replayed_into_an_empty_doc_reproduces_the_document() {

+ let source = doc_with(1);

+ let update = std::fs::read("../fixtures/vectors/fragment/simple.bin").unwrap();

+ source

+ .transact_mut()

+ .apply_update(yrs::Update::decode_v1(&update).unwrap())

+ .unwrap();

+ let snap = snapshot_bytes(&source);

+ let rebuilt = doc_with(2);

+ rebuilt

+ .transact_mut()

+ .apply_update(yrs::Update::decode_v1(&snap).unwrap())

+ .unwrap();

+ assert_eq!(fragment_text(&rebuilt), fragment_text(&source));

+ assert!(

+ !fragment_text(&rebuilt).is_empty(),

+ "fixture produced no text"

+ );

diff --git a/internal/core/core.go b/internal/core/core.go

index 188fc04..10bcea4 100644

--- a/internal/core/core.go

+++ b/internal/core/core.go

@@ -32,15 +32,18 @@

// Spec §5.2 describes recovery as rebuilding "from its latest snapshot" —

// a persisted, compacted state plus a bounded tail of subsequent updates.

-// ABI v1 exports no state-encode function (nothing here can ask the guest

-// "give me your current state as a single update"), so there is no

-// snapshot to load. What this package does instead: every Handle keeps the

-// updates it has itself applied successfully, in memory, for the life of

-// the Handle, and a discarded instance is rebuilt by replaying that entire

-// log from an empty document. This is a deliberate substitution, not an

-// oversight — it is what ABI v1 makes possible — and it is why the log

-// needs its own bound (maxRetainedUpdates below): without real snapshot

-// support, "replay everything" is the only recovery this package can do,

+// As of ABI v4 the guest module can encode its whole state as one update

+// (docs_core_snapshot_len / docs_core_snapshot_read), but this package

+// does not yet call them or persist anything — that wiring belongs to

+// the snapshot-storage task this one sets up for. Until then, every

+// Handle keeps the updates it has itself applied successfully, in

+// memory, for the life of the Handle, and a discarded instance is

+// rebuilt by replaying that entire log from an empty document. This is

+// a deliberate substitution, not an oversight — it is what the absence

+// of snapshot *storage* (not, any longer, of the export itself) makes

+// necessary — and it is why the log still needs its own bound

+// (maxRetainedUpdates below): without storage wired up, "replay

+// everything" remains the only recovery this package can do,

// so the log cannot be allowed to grow without limit.

package core

@@ -62,7 +65,22 @@ var moduleWASM []byte

// anything else is refused at load, so a stale .wasm (see

// scripts/build-wasm.sh and the copy step in Task 6's brief) fails loudly

// rather than behaving strangely.

-const abiVersion = 3

+//

+// Bumped to 4 alongside core/src/lib.rs's docs_core_abi_version for the

+// snapshot export pair (docs_core_snapshot_len / docs_core_snapshot_read)

+// that encodes a document's whole state as one update — see that

+// function's doc comment for why an added export forces a version bump

+// here even though it is, strictly, backwards compatible on the wasm

+// side: the module and this binary ship embedded together with no

+// version skew to tolerate, so the bump exists to catch a stale .wasm

+// at load time instead of "unknown export" deep inside a request.

+const abiVersion = 4

+// ABIVersion is abiVersion, exported for callers outside this package

+// (snapshot storage and its tests) that need to know which ABI contract

+// this host was built against. It is the same constant, not a second

+// number that could drift from it.

+const ABIVersion = abiVersion

// defaultGuestCallTimeout bounds every single call into the wasm module,

// unless overridden with WithGuestCallTimeout.

@@ -86,16 +104,17 @@ const defaultGuestCallTimeout = 1 * time.Second

// timed-out call forcing one — would grow without limit (fix-round-1

// finding 5).

-// The real fix is snapshot support: an ABI v2 export that encodes a

-// document's current state as one update (yrs supports this internally;

-// nothing in core/src/abi.rs exposes it yet), so rebuild could restore a

+// The real fix is snapshot support: ABI v4 now exports a state-encode

+// pair (docs_core_snapshot_len / docs_core_snapshot_read) that encodes a

+// document's current state as one update, so rebuild could restore a

// compacted snapshot plus only the updates since it, matching spec §5.2 and

// the doc_snapshots table spec §4.1 describes — bounded, cheap recovery

-// regardless of a document's total history. That belongs with the

-// snapshot/compaction work, not here.

+// regardless of a document's total history. Calling that export and

+// persisting the result belongs with the snapshot/compaction work, not

+// here — this package does not do either yet.

-// Until then, this package does the safe thing available under ABI v1:

-// once a document's log is full, Apply refuses to accept further updates

+// Until then, this package does the safe thing available without that

+// storage wired up: once a document's log is full, Apply refuses to accept further updates

// at all (ErrUpdateLogFull), rather than either growing the log without

// bound or silently dropping old entries — dropping old entries would make

// a later rebuild replay an incomplete history and produce silently wrong

@@ -117,6 +136,11 @@ const defaultGuestCallTimeout = 1 * time.Second

// comment's own snapshot support, not a bigger number here.

const maxRetainedUpdates = 4096

+// MaxRetainedUpdates is maxRetainedUpdates, exported for callers outside

+// this package (snapshot storage and its tests) that need the same cap.

+// It is the same constant, not a second number that could drift from it.

+const MaxRetainedUpdates = maxRetainedUpdates

// ErrUpdateLogFull is returned by Handle.Apply once a document's retained

// update log has reached maxRetainedUpdates. See that constant's doc

// comment: this is a deliberate, visible failure standing in for

diff --git a/internal/core/testdata/docs_core.wasm b/internal/core/testdata/docs_core.wasm

index 45e7c36..419de9d 100755

Binary files a/internal/core/testdata/docs_core.wasm and b/internal/core/testdata/docs_core.wasm differ