amadan

oficina / docs Public

app: documents in SQLite, text from the CRDT core

Content is not a column -- Text replays the update log through the
wazero core. GUID and client id are assigned by the app and stored,
never drawn from yrs, which is the Go side of the rule clippy.toml
enforces in Rust.
Paul Campbell pushed by paul@keymail.dev be96717906e2c23a8b8f55ca77fee54d0e61dda3 parent 032a355
7 files changed, +444 −10
  • go.mod +2 −2
  • internal/docs/app.go +40 −8
  • internal/docs/migrations/0001_documents.sql +18 −0
  • internal/docs/models.go +68 −0
  • internal/docs/store.go +232 −0
  • internal/docs/store_test.go +70 −0
  • internal/docs/testing.go +14 −0

Intent

full ledger: app-first-screens

diff --git a/go.mod b/go.mod
index d17b7bd..469fedf 100644
--- a/go.mod
+++ b/go.mod
@@ -5,12 +5,13 @@ go 1.26.5
require (
github.com/carlosframework/rastrillo v0.21.0
github.com/go-chi/chi/v5 v5.3.2
+ github.com/google/uuid v1.6.0
github.com/tetratelabs/wazero v1.12.0
+ gorm.io/gorm v1.31.2
)
require (
github.com/dustin/go-humanize v1.0.1 // indirect
- github.com/google/uuid v1.6.0 // indirect
github.com/jinzhu/inflection v1.0.0 // indirect
github.com/jinzhu/now v1.1.5 // indirect
github.com/mattn/go-isatty v0.0.20 // indirect
@@ -18,7 +19,6 @@ require (
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
golang.org/x/sys v0.46.0 // indirect
golang.org/x/text v0.20.0 // indirect
- gorm.io/gorm v1.31.2 // indirect
gorm.io/plugin/dbresolver v1.6.2 // indirect
modernc.org/libc v1.74.1 // indirect
modernc.org/mathutil v1.7.1 // indirect
diff --git a/internal/docs/app.go b/internal/docs/app.go
index 054384a..08a7738 100644
--- a/internal/docs/app.go
+++ b/internal/docs/app.go
@@ -12,6 +12,7 @@ package docs
import (
"context"
+ "errors"
"fmt"
"log/slog"
"net/http"
@@ -19,6 +20,8 @@ import (
"github.com/carlosframework/rastrillo/db"
"github.com/carlosframework/rastrillo/migrate"
"github.com/go-chi/chi/v5"
+
+ "github.com/carlosframework/docs/internal/core"
)
// Config configures New.
@@ -48,12 +51,14 @@ type App struct {
db *db.DB
mux *http.ServeMux
logger *slog.Logger
+ core *core.Runtime
+ store *Store
}
-// BootSchema is everything New applies at boot, in apply order. Empty
-// until Task 2 adds models.go; migrate.Merge(sessions.Schema, ...)
-// lands here once auth does.
-var BootSchema = migrate.Merge()
+// BootSchema is everything New applies at boot, in apply order.
+// migrate.Merge(sessions.Schema, ...) adds another set here once auth
+// does; for now it's just this app's own Schema (models.go).
+var BootSchema = migrate.Merge(Schema)
// New opens the database, applies migrations and wires the router. It
// is rastrillo SKILL.md's app.go shape: db.Open for the SQLite handle,
@@ -81,8 +86,19 @@ func New(cfg Config) (*App, error) {
return nil, fmt.Errorf("docs: apply migrations: %w", err)
}
+ // One Runtime for the process (core.New compiles the wasm module
+ // once — expensive; instantiating a per-document Handle from it is
+ // not), shared by every document's Store.replay.
+ rt, err := core.New(context.Background())
+ if err != nil {
+ d.Close()
+ return nil, fmt.Errorf("docs: start core runtime: %w", err)
+ }
+ store := NewStore(d.G, rt)
+
render, err := newRenderer()
if err != nil {
+ rt.Close(context.Background())
d.Close()
return nil, fmt.Errorf("docs: build renderer: %w", err)
}
@@ -96,9 +112,14 @@ func New(cfg Config) (*App, error) {
mux := http.NewServeMux()
mux.Handle("/", r)
- return &App{cfg: cfg, db: d, mux: mux, logger: logger}, nil
+ return &App{cfg: cfg, db: d, mux: mux, logger: logger, core: rt, store: store}, nil
}
+// Store returns the app's document store — models.go's Document,
+// store.go's Store — over the app's migrated database and shared
+// core.Runtime.
+func (a *App) Store() *Store { return a.store }
+
// Handler is the app's router: rastrillo.Options.Mux for real serving
// (cmd/docs/main.go), and — per rastrillo's own doc comment on
// Handler/Serve — exactly what a test harness should exercise directly,
@@ -108,6 +129,17 @@ func New(cfg Config) (*App, error) {
// and matches SKILL.md's own App(...) (*http.ServeMux, error) shape.
func (a *App) Handler() *http.ServeMux { return a.mux }
-// Close releases the database handle. NewForTest registers this as a
-// t.Cleanup.
-func (a *App) Close() error { return a.db.Close() }
+// Close releases the database handle and the core runtime's compiled
+// module. NewForTest registers this as a t.Cleanup.
+func (a *App) Close() error {
+ var errs []error
+ if a.core != nil {
+ if err := a.core.Close(context.Background()); err != nil {
+ errs = append(errs, err)
+ }
+ }
+ if err := a.db.Close(); err != nil {
+ errs = append(errs, err)
+ }
+ return errors.Join(errs...)
+}
diff --git a/internal/docs/migrations/0001_documents.sql b/internal/docs/migrations/0001_documents.sql
new file mode 100644
index 0000000..936b66d
--- /dev/null
+++ b/internal/docs/migrations/0001_documents.sql
@@ -0,0 +1,18 @@
+CREATE TABLE IF NOT EXISTS documents (
+ id INTEGER PRIMARY KEY,
+ sqid TEXT NOT NULL UNIQUE,
+ title TEXT NOT NULL,
+ guid TEXT NOT NULL,
+ client_id INTEGER NOT NULL,
+ created_at DATETIME NOT NULL,
+ updated_at DATETIME NOT NULL,
+ deleted_at DATETIME
+);
+
+CREATE TABLE IF NOT EXISTS doc_updates (
+ id INTEGER PRIMARY KEY,
+ doc_id INTEGER NOT NULL,
+ seq INTEGER NOT NULL,
+ update_bin BLOB NOT NULL,
+ created_at DATETIME NOT NULL
+);
diff --git a/internal/docs/models.go b/internal/docs/models.go
new file mode 100644
index 0000000..d1447c9
--- /dev/null
+++ b/internal/docs/models.go
@@ -0,0 +1,68 @@
+package docs
+
+import (
+ "embed"
+ "time"
+
+ "github.com/carlosframework/rastrillo/migrate"
+)
+
+// migrationFS embeds this app's own numbered migrations — per
+// rastrillo SKILL.md's app shape ("Two *migrate.Set values live in
+// internal/<app>/migrations.go, beside models.go"; kept in this file
+// instead of a separate migrations.go because this task's file list
+// is models.go, store.go, store_test.go and a fourth file would only
+// split one small declaration across two places for no benefit).
+//
+//go:embed migrations/*.sql
+var migrationFS embed.FS
+
+// Schema is this app's own migrations — spec §4's documents and
+// doc_updates tables, only what this plan needs. It is idempotent
+// (CREATE TABLE IF NOT EXISTS today; later plans add further numbered
+// files, additive ALTERs, never edits to 0001) per rastrillo's
+// migration pattern.
+var Schema = migrate.MustFromFS(migrationFS, "docs")
+
+// Document is a document's row. Its text is deliberately not a
+// column: GUID and ClientID are what let Store open the real CRDT
+// core (internal/core) and replay doc_updates into it, which is where
+// the text actually comes from — see Store.Text.
+//
+// GUID and ClientID are assigned by the app (Store.Create) and stored
+// here; they are never drawn from yrs itself — on wasm the CRDT
+// library's entropy is a constant seed, so a value it generates is
+// identical in every process, and a repeated client id or GUID
+// corrupts the document. This is the Go side of the rule clippy.toml
+// enforces in Rust (disallowed-methods on yrs::Doc::new /
+// Doc::with_client_id).
+type Document struct {
+ ID int64 `gorm:"primaryKey"`
+ Sqid string `gorm:"column:sqid;uniqueIndex;not null"`
+ Title string `gorm:"column:title;not null"`
+
+ GUID string `gorm:"column:guid;not null"`
+ ClientID uint64 `gorm:"column:client_id;not null"`
+
+ CreatedAt time.Time `gorm:"column:created_at;not null"`
+ UpdatedAt time.Time `gorm:"column:updated_at;not null"`
+ DeletedAt *time.Time `gorm:"column:deleted_at"`
+}
+
+// DocUpdate is one Yjs update in a document's log, applied in Seq
+// order to rebuild the document's CRDT state (Store.replay).
+//
+// UpdateBin is plaintext in this table. Sealing it (spec §2 — per-row
+// AES-GCM, same as doc titles and everything else that is document
+// content) is out of scope for this plan; this comment exists so the
+// next reader does not mistake that for a decision that it stays
+// plaintext.
+type DocUpdate struct {
+ ID int64 `gorm:"primaryKey"`
+ DocID int64 `gorm:"column:doc_id;not null;index"`
+ Seq int64 `gorm:"column:seq;not null"`
+
+ UpdateBin []byte `gorm:"column:update_bin;not null"`
+
+ CreatedAt time.Time `gorm:"column:created_at;not null"`
+}
diff --git a/internal/docs/store.go b/internal/docs/store.go
new file mode 100644
index 0000000..175a441
--- /dev/null
+++ b/internal/docs/store.go
@@ -0,0 +1,232 @@
+package docs
+
+import (
+ "context"
+ crand "crypto/rand"
+ "errors"
+ "fmt"
+ mrand "math/rand/v2"
+ "time"
+
+ "github.com/google/uuid"
+ "gorm.io/gorm"
+
+ "github.com/carlosframework/docs/internal/core"
+)
+
+// Store is documents' persistence: rows in SQLite for identity and
+// the update log, and the document's actual text read back through
+// the real CRDT core rather than any column here — see Text.
+type Store struct {
+ db *gorm.DB
+ core *core.Runtime
+}
+
+// NewStore builds a Store over an already-migrated *gorm.DB and a
+// shared *core.Runtime — one Runtime compiles the wasm module once
+// and is reused for every document (core.New's doc comment); each
+// call into it below opens its own per-document instance (a
+// *core.Handle) and closes it again.
+func NewStore(db *gorm.DB, rt *core.Runtime) *Store {
+ return &Store{db: db, core: rt}
+}
+
+// sqidAlphabet avoids visually ambiguous characters (0/O, 1/l/I) —
+// sqids appear in URLs a person may read aloud or retype.
+const sqidAlphabet = "23456789abcdefghjkmnpqrstuvwxyz"
+
+const sqidLength = 10
+
+// newSqid generates a random opaque document identifier, distinct
+// from the row's integer id, safe to expose in a URL. len(sqidAlphabet)
+// is 32, and 256 is an exact multiple of 32, so taking each random
+// byte mod 32 introduces no modulo bias.
+func newSqid() (string, error) {
+ raw := make([]byte, sqidLength)
+ if _, err := crand.Read(raw); err != nil {
+ return "", fmt.Errorf("docs: generate sqid: %w", err)
+ }
+ out := make([]byte, sqidLength)
+ for i, b := range raw {
+ out[i] = sqidAlphabet[int(b)%len(sqidAlphabet)]
+ }
+ return string(out), nil
+}
+
+// maxSqidAttempts bounds Create's retry loop on a sqid collision. At
+// sqidLength characters over a 32-symbol alphabet, a collision is
+// astronomically unlikely; this exists only so a UNIQUE violation is
+// handled rather than surprising, not because collisions are
+// expected.
+const maxSqidAttempts = 5
+
+// Create makes a new document: a GUID and a client id assigned here
+// by the app (never drawn from yrs — see Document's doc comment) and
+// stored, an app-generated sqid, and the given title. It does not
+// touch the CRDT core at all: an empty document needs no update log
+// entry, and Text/ApplyUpdate are what open a core handle when there
+// is work for one to do.
+func (s *Store) Create(ctx context.Context, title string) (*Document, error) {
+ guid := uuid.NewString()
+ // math/rand/v2's Int64 returns a non-negative value in [0, 1<<63),
+ // so converting to uint64 never wraps — the brief's "a counter or
+ // random int64 from math/rand/v2".
+ clientID := uint64(mrand.Int64())
+ now := time.Now().UTC()
+
+ var lastErr error
+ for attempt := 0; attempt < maxSqidAttempts; attempt++ {
+ sqid, err := newSqid()
+ if err != nil {
+ return nil, err
+ }
+ doc := &Document{
+ Sqid: sqid,
+ Title: title,
+ GUID: guid,
+ ClientID: clientID,
+ CreatedAt: now,
+ UpdatedAt: now,
+ }
+ err = s.db.WithContext(ctx).Create(doc).Error
+ if err == nil {
+ return doc, nil
+ }
+ lastErr = err
+ if !errors.Is(err, gorm.ErrDuplicatedKey) {
+ break
+ }
+ }
+ return nil, fmt.Errorf("docs: create document: %w", lastErr)
+}
+
+// List returns every non-deleted document, newest first.
+func (s *Store) List(ctx context.Context) ([]*Document, error) {
+ var docs []*Document
+ err := s.db.WithContext(ctx).
+ Where("deleted_at IS NULL").
+ Order("created_at DESC, id DESC").
+ Find(&docs).Error
+ if err != nil {
+ return nil, fmt.Errorf("docs: list documents: %w", err)
+ }
+ return docs, nil
+}
+
+// Get looks up a document by its sqid.
+func (s *Store) Get(ctx context.Context, sqid string) (*Document, error) {
+ var doc Document
+ err := s.db.WithContext(ctx).
+ Where("sqid = ? AND deleted_at IS NULL", sqid).
+ First(&doc).Error
+ if err != nil {
+ return nil, fmt.Errorf("docs: get document %q: %w", sqid, err)
+ }
+ return &doc, nil
+}
+
+// replay opens a fresh core.Handle for doc (guid and client id come
+// from the stored row, never from yrs — see Document) and applies
+// every update in doc_updates, in seq order, rebuilding the
+// document's CRDT state from scratch. The caller owns the returned
+// Handle and must Close it.
+//
+// This is a deliberate O(n) full replay on every call, matching
+// internal/core's own recovery strategy (its package doc comment:
+// "Rebuilding is log replay, not snapshot restore" — ABI v1 exports
+// no state-encode function, so there is no snapshot to load instead).
+// That is fine at this plan's scale (a handful of updates per
+// document); a document with a long history needs compaction
+// (ErrUpdateLogFull's doc comment) or ABI v2 snapshot support,
+// neither of which exists yet — replaying long histories here would
+// be papering over that gap rather than living within it.
+func (s *Store) replay(ctx context.Context, doc *Document) (*core.Handle, error) {
+ h, err := s.core.NewDoc(ctx, doc.GUID, doc.ClientID)
+ if err != nil {
+ return nil, fmt.Errorf("docs: open core handle for document %d: %w", doc.ID, err)
+ }
+
+ var updates []DocUpdate
+ if err := s.db.WithContext(ctx).
+ Where("doc_id = ?", doc.ID).
+ Order("seq ASC").
+ Find(&updates).Error; err != nil {
+ h.Close(ctx)
+ return nil, fmt.Errorf("docs: load updates for document %d: %w", doc.ID, err)
+ }
+
+ for _, u := range updates {
+ if err := h.Apply(ctx, u.UpdateBin); err != nil {
+ h.Close(ctx)
+ // Surfaced, not swallowed: whether this is a clean
+ // rejection, core.ErrUpdateLogFull, or a *core.RecoveredError
+ // (the specific update's outcome unknown, never retried —
+ // see Handle.Apply's doc comment), the log this Store
+ // itself wrote can no longer be trusted to reproduce the
+ // document, and callers must see that rather than silently
+ // getting a wrong or truncated text back.
+ return nil, fmt.Errorf("docs: replay update seq %d for document %d: %w", u.Seq, doc.ID, err)
+ }
+ }
+ return h, nil
+}
+
+// Text returns doc's current text — read from the real CRDT core by
+// replaying its update log, never from a database column. Sealing
+// (spec §2 — per-row encryption of update_bin, same as document
+// titles) is out of scope for this plan; this plaintext replay is not
+// a decision that content stays unsealed.
+func (s *Store) Text(ctx context.Context, doc *Document) (string, error) {
+ h, err := s.replay(ctx, doc)
+ if err != nil {
+ return "", err
+ }
+ defer h.Close(ctx)
+
+ text, err := h.Text(ctx)
+ if err != nil {
+ return "", fmt.Errorf("docs: read text for document %d: %w", doc.ID, err)
+ }
+ return text, nil
+}
+
+// ApplyUpdate applies a new Yjs update to doc: it replays the
+// existing log to rebuild current state, applies update against that
+// live core instance, and only once the core has accepted it does it
+// persist the update as the next row in doc_updates. A rejection from
+// the core (a clean rejection, core.ErrUpdateLogFull, or a
+// *core.RecoveredError) is returned as-is and the update is never
+// persisted — see the errors.Is-friendly wrapping in replay's doc
+// comment and Handle.Apply's for why an uncertain outcome must not be
+// written down as a fact.
+func (s *Store) ApplyUpdate(ctx context.Context, doc *Document, update []byte) error {
+ h, err := s.replay(ctx, doc)
+ if err != nil {
+ return err
+ }
+ defer h.Close(ctx)
+
+ if err := h.Apply(ctx, update); err != nil {
+ return fmt.Errorf("docs: apply update to document %d: %w", doc.ID, err)
+ }
+
+ var maxSeq int64
+ if err := s.db.WithContext(ctx).
+ Model(&DocUpdate{}).
+ Where("doc_id = ?", doc.ID).
+ Select("COALESCE(MAX(seq), 0)").
+ Scan(&maxSeq).Error; err != nil {
+ return fmt.Errorf("docs: determine next seq for document %d: %w", doc.ID, err)
+ }
+
+ row := &DocUpdate{
+ DocID: doc.ID,
+ Seq: maxSeq + 1,
+ UpdateBin: update,
+ CreatedAt: time.Now().UTC(),
+ }
+ if err := s.db.WithContext(ctx).Create(row).Error; err != nil {
+ return fmt.Errorf("docs: persist update for document %d: %w", doc.ID, err)
+ }
+ return nil
+}
diff --git a/internal/docs/store_test.go b/internal/docs/store_test.go
new file mode 100644
index 0000000..a21471b
--- /dev/null
+++ b/internal/docs/store_test.go
@@ -0,0 +1,70 @@
+package docs_test
+
+import (
+ "context"
+ "testing"
+
+ "github.com/carlosframework/docs/internal/docs"
+)
+
+func TestCreateListGetRoundTrip(t *testing.T) {
+ app, err := docs.NewForTest(t)
+ if err != nil {
+ t.Fatalf("NewForTest: %v", err)
+ }
+ ctx := context.Background()
+
+ made, err := app.Store().Create(ctx, "First document")
+ if err != nil {
+ t.Fatalf("Create: %v", err)
+ }
+ if made.Sqid == "" {
+ t.Fatal("Create returned an empty sqid")
+ }
+ if made.GUID == "" || made.ClientID == 0 {
+ t.Fatal("Create must assign a GUID and client id, never draw them from yrs")
+ }
+
+ list, err := app.Store().List(ctx)
+ if err != nil {
+ t.Fatalf("List: %v", err)
+ }
+ if len(list) != 1 || list[0].Title != "First document" {
+ t.Fatalf("List = %+v, want one document titled First document", list)
+ }
+
+ got, err := app.Store().Get(ctx, made.Sqid)
+ if err != nil {
+ t.Fatalf("Get: %v", err)
+ }
+ if got.ID != made.ID {
+ t.Errorf("Get returned id %d, want %d", got.ID, made.ID)
+ }
+}
+
+func TestTextComesFromTheCore(t *testing.T) {
+ app, err := docs.NewForTest(t)
+ if err != nil {
+ t.Fatalf("NewForTest: %v", err)
+ }
+ ctx := context.Background()
+
+ doc, err := app.Store().Create(ctx, "With content")
+ if err != nil {
+ t.Fatalf("Create: %v", err)
+ }
+
+ // The "hello" fixture is a real Yjs update produced by browser yjs.
+ update := docs.ReadFixture(t, "hello")
+ if err := app.Store().ApplyUpdate(ctx, doc, update); err != nil {
+ t.Fatalf("ApplyUpdate: %v", err)
+ }
+
+ text, err := app.Store().Text(ctx, doc)
+ if err != nil {
+ t.Fatalf("Text: %v", err)
+ }
+ if text != "hello world" {
+ t.Errorf("Text = %q, want %q — the text must come from the CRDT, not a column", text, "hello world")
+ }
+}
diff --git a/internal/docs/testing.go b/internal/docs/testing.go
index c1cc6ad..ad91d27 100644
--- a/internal/docs/testing.go
+++ b/internal/docs/testing.go
@@ -1,6 +1,7 @@
package docs
import (
+ "os"
"path/filepath"
"testing"
)
@@ -33,3 +34,16 @@ func NewForTest(t *testing.T) (*App, error) {
})
return app, nil
}
+
+// ReadFixture loads fixtures/data/<name>.update.bin — a real Yjs
+// update produced by browser yjs (fixtures/generate.mjs), never
+// hand-edited. Not in a _test.go file for the same reason as
+// NewForTest: other packages' tests may need it too.
+func ReadFixture(t *testing.T, name string) []byte {
+ t.Helper()
+ b, err := os.ReadFile(filepath.Join("..", "..", "fixtures", "data", name+".update.bin"))
+ if err != nil {
+ t.Fatalf("docs.ReadFixture(%q): %v", name, err)
+ }
+ return b
+}