amadan

oficina / docs Public

Merge branch 'overnight-content-sealing' into main

claude pushed by paul@keymail.dev 87662f5b246004aca604a4b040cb6691cfd7f130 parent 3d5dfadparent df7db78
22 files changed, +1526 −24
  • PRODUCT.md +1 −4
  • cmd/docs/content_maintenance.go +26 −0
  • cmd/docs/content_maintenance_test.go +53 −0
  • cmd/docs/main.go +8 −0
  • docs/superpowers/specs/2026-09-07-content-sealing.md +93 −0
  • internal/docs/app.go +22 −1
  • internal/docs/content_cipher.go +95 −0
  • internal/docs/content_cipher_test.go +92 −0
  • internal/docs/content_maintenance.go +52 −0
  • internal/docs/content_maintenance_test.go +96 −0
  • internal/docs/content_migration.go +147 −0
  • internal/docs/content_migration_test.go +169 −0
  • internal/docs/content_recovery_test.go +133 −0
  • internal/docs/content_rows.go +47 −0
  • internal/docs/content_runtime.go +67 −0
  • internal/docs/content_runtime_test.go +329 −0
  • internal/docs/content_scrub.go +52 −0
  • internal/docs/handlers.go +8 −0
  • internal/docs/migrations/0007_content_encoding.sql +10 −0
  • internal/docs/models.go +5 −13
  • internal/docs/snapshots.go +10 −0
  • internal/docs/store.go +11 −6

CI passed — run details

diff --git a/PRODUCT.md b/PRODUCT.md
index f3ac319..f95de50 100644
--- a/PRODUCT.md
+++ b/PRODUCT.md
@@ -71,10 +71,7 @@ its foundations:
held by the running instance, enforced by a test that greps raw SQLite bytes
and bucket objects for fixture plaintext, and it enumerates the six things
that *do* leak (§2.1) rather than eliding them.
- **None of that is implemented today.** `Store.Text`
- (`internal/docs/store.go`) says so in its own doc comment — sealing is out
- of scope for the plan that built it — and no plaintext-grep test exists in
- the repo. Do not describe this instance as encrypting content at rest, and
+ Do not describe this instance as encrypting content at rest, and
do not let this entry be read as a shipped property; it is the position the
product is being built toward, and the thing that makes the position
defensible is the leak list, which is honest about what the trade costs.
diff --git a/cmd/docs/content_maintenance.go b/cmd/docs/content_maintenance.go
new file mode 100644
index 0000000..f9a2618
--- /dev/null
+++ b/cmd/docs/content_maintenance.go
@@ -0,0 +1,26 @@
+package main
+
+import (
+ "context"
+ "flag"
+ "fmt"
+ "io"
+ "log/slog"
+
+ "github.com/carlosframework/docs/internal/docs"
+)
+
+func runContentMaintenance(ctx context.Context, args []string, getenv func(string) string, logger *slog.Logger) error {
+ flags := flag.NewFlagSet("docs-seal", flag.ContinueOnError)
+ flags.SetOutput(io.Discard)
+ path := flags.String("db", "", "")
+ fingerprint := flags.String("expect-fingerprint", "", "")
+ scrub := flags.Bool("scrub", false, "")
+ if err := flags.Parse(args); err != nil {
+ return err
+ }
+ if flags.NArg() != 0 || *path == "" || *fingerprint == "" {
+ return fmt.Errorf("docs-seal: -db, -expect-fingerprint")
+ }
+ return docs.MaintainContent(ctx, docs.ContentMaintenanceConfig{DBPath: *path, ContentKey: getenv("DOCS_CONTENT_KEY"), ExpectedFingerprint: *fingerprint, Scrub: *scrub, Logger: logger})
+}
diff --git a/cmd/docs/content_maintenance_test.go b/cmd/docs/content_maintenance_test.go
new file mode 100644
index 0000000..3cfce77
--- /dev/null
+++ b/cmd/docs/content_maintenance_test.go
@@ -0,0 +1,53 @@
+package main
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/hex"
+ "log/slog"
+ "path/filepath"
+ "testing"
+
+ "github.com/carlosframework/docs/internal/docs"
+)
+
+func TestContentMaintenanceCommandActivatesExistingDatabase(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "docs.db")
+ key := bytes.Repeat([]byte{91}, 32)
+ cfg := docs.Config{DBPath: path, Env: "development", ContentKey: base64.StdEncoding.EncodeToString(key)}
+ app, err := docs.New(cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := app.Close(); err != nil {
+ t.Fatal(err)
+ }
+ fingerprint := sha256.Sum256(key)
+ env := func(name string) string {
+ if name != "DOCS_CONTENT_KEY" {
+ t.Fatalf("unexpected env: %s", name)
+ }
+ return cfg.ContentKey
+ }
+ if err := runContentMaintenance(context.Background(), []string{"-db", path, "-expect-fingerprint", hex.EncodeToString(fingerprint[:]), "-scrub"}, env, slog.Default()); err != nil {
+ t.Fatal(err)
+ }
+ cfg.ContentKey = base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{92}, 32))
+ app, err = docs.New(cfg)
+ if app != nil {
+ app.Close()
+ }
+ if err == nil {
+ t.Fatal("maintenance did not bind database to key")
+ }
+}
+
+func TestContentMaintenanceCommandRequiresExplicitInputs(t *testing.T) {
+ for _, args := range [][]string{nil, {"-db", "somewhere"}, {"-expect-fingerprint", "something"}, {"-deactivate"}, {"-db", "somewhere", "-expect-fingerprint", "something", "extra"}} {
+ if err := runContentMaintenance(context.Background(), args, func(string) string { t.Fatal("invalid CLI reached key loading"); return "" }, slog.Default()); err == nil {
+ t.Fatalf("accepted args: %v", args)
+ }
+ }
+}
diff --git a/cmd/docs/main.go b/cmd/docs/main.go
index 7f68336..69cf7d6 100644
--- a/cmd/docs/main.go
+++ b/cmd/docs/main.go
@@ -135,6 +135,13 @@ func checkSeedGate(seed bool, env string) error {
func main() {
logger := slog.Default()
+ if len(os.Args) > 1 && os.Args[1] == "docs-seal" {
+ if err := runContentMaintenance(context.Background(), os.Args[2:], os.Getenv, logger); err != nil {
+ logger.Error("docs: new", "err", err)
+ os.Exit(1)
+ }
+ return
+ }
// -env (chooseEnv above; $DOCS_ENV is consulted too, but never for
// "development" — see chooseEnv's own doc comment, fix round 2)
@@ -264,6 +271,7 @@ func main() {
// already owns.
cfg := docs.Config{
DBPath: opts.DBPath,
+ ContentKey: os.Getenv("DOCS_CONTENT_KEY"),
Env: env,
Origin: origin,
InstanceKey: instanceKey,
diff --git a/docs/superpowers/specs/2026-09-07-content-sealing.md b/docs/superpowers/specs/2026-09-07-content-sealing.md
new file mode 100644
index 0000000..2828f5c
--- /dev/null
+++ b/docs/superpowers/specs/2026-09-07-content-sealing.md
@@ -0,0 +1,93 @@
+# Document body sealing implementation spec
+
+Status: approved for implementation by Claude Fable 5.1
+(`claude-fable-5-1`), session
+`8aa40554-38b1-4597-9cb8-439ab594e268`. The final rollout amendment and
+approval are recorded in Amadan Docs discussion 39. Whole-branch review,
+exact-head CI, release integration, canary and deployment evidence are
+separate gates.
+
+## Scope and encoding
+
+Seal `doc_updates.update_bin` and `doc_snapshots.state`. Titles and other
+metadata are outside this slice. Historical plaintext backups remain
+preserved and are not claimed encrypted by this change.
+
+Use a separate `DOCS_CONTENT_KEY`: strict standard-base64 encoding of 32
+random bytes. Require it in production before opening the database, even
+before activation. Only explicit development mode may use the fixed public
+development key; maintenance always validates as production.
+
+Encoding 1 uses AES-256-GCM with a random 12-byte nonce prepended to the
+ciphertext and tag. Associated data binds the fixed content domain, row
+kind, immutable document UUID, sequence and snapshot ABI version. Allocate
+the sequence inside the writer transaction before sealing the update.
+Never send ciphertext to the CRDT core or sync clients. Decrypt and
+validate all response rows before writing HTTP success headers.
+
+The additive migration creates encoding columns defaulting to 0 and a
+singleton activation marker containing encoding, key fingerprint,
+authenticated keycheck and scrubbed bookkeeping. Keycheck binds a fixed
+sentinel and remains meaningful for an empty database. Wrong-key errors
+are distinct from authentication failures and never include content bytes.
+
+## Runtime states
+
+A database without the marker is in legacy compatibility mode. Refuse
+startup if it already contains nonzero encoding rows. Log an Error on
+every legacy boot with structured encoding and key-fingerprint fields.
+Legacy reads accept only encoding 0. Both update and snapshot inserts
+re-read the marker through the same writer transaction as the insert and
+refuse if activation has occurred. A process never switches modes live.
+
+A database with the marker verifies its keycheck and refuses startup if
+an EXISTS query finds any update or snapshot with encoding other than 1.
+Activated reads reject other encodings and authenticate every body used.
+An unusable snapshot may be abandoned for the retained update log; it is
+an accelerator and is never the only copy of document history.
+
+## Explicit maintenance
+
+Only the `docs-seal` subcommand invokes conversion or scrubbing. It requires
+an existing regular database path and the expected key fingerprint. Read
+the key only from `DOCS_CONTENT_KEY`. Reuse the exact server `BootSchema`.
+Never resolve serving options, bind a listener or start replication.
+There is no deactivate or marker-removal operation.
+
+Conversion is one transaction. Process bounded batches, bind bodies to
+their stored document UUID, verify already-sealed rows and reject unknown
+encodings or orphaned rows. On first conversion write the keycheck marker
+atomically with the converted rows. On explicit repair of an activated
+mixed database, convert remaining encoding-0 rows while leaving the marker,
+fingerprint and keycheck unchanged. Clear only scrubbed bookkeeping when
+new legacy bytes have been converted. Any failure rolls back all changes.
+
+Scrubbing is an explicit operation after conversion. The operator stops
+the server and replication first. Set scrubbed false, use a pinned writer
+connection, check the complete TRUNCATE-checkpoint result, VACUUM, check a
+second TRUNCATE checkpoint, and set scrubbed true only on success. A busy
+checkpoint is failure. Never put conversion or VACUUM on normal startup.
+Every restored copy needs another explicit scrub; its stored scrubbed bit
+is not evidence about the restored file's pages or WAL.
+
+## Rollout constraints
+
+Deploy and smoke-test the compatibility-capable release with the real
+production key before activation. Retain that exact release as the rollback
+target. Verify key fingerprint and backup/key recovery before conversion.
+Stop serving and replication for maintenance, then restart the same
+compatibility-capable release after activation. Never roll back to an older
+plaintext-only binary once sealed rows exist. Preserve historical backups.
+A restored pre-conversion backup is visibly legacy and needs activation;
+it must never be silently presented as encrypted.
+
+## Required evidence
+
+Use actual Yjs fixtures and real SQLite. Cover authenticated row identity,
+wrong keys and tampering; encrypted HTTP read/write and snapshots; stale
+legacy reads and writes; both tables' mixed-state startup checks; explicit
+maintenance inputs; atomic repair and marker preservation; raw database
+and WAL canaries; incomplete checkpoints; process-exit recovery; and an
+independent restored-copy rehearsal. Observe meaningful tests fail when
+their safeguards are removed, then restore the code. Full repository CI
+and an independent complete-diff review remain required before merge.
diff --git a/internal/docs/app.go b/internal/docs/app.go
index 5308573..a5ddb45 100644
--- a/internal/docs/app.go
+++ b/internal/docs/app.go
@@ -33,7 +33,8 @@ import (
// Config configures New.
type Config struct {
// DBPath is the SQLite file New opens via rastrillo's db package.
- DBPath string
+ DBPath string
+ ContentKey string
// Env is "development" or "production" ("" defaults to
// "production"). It gates two fail-closed defaults below — Origin
@@ -361,6 +362,11 @@ func New(cfg Config) (*App, error) {
return nil, err
}
+ cipher, err := newContentCipher(cfg.ContentKey, cfg.Env)
+ if err != nil {
+ return nil, err
+ }
+
d, err := db.Open(cfg.DBPath, logger)
if err != nil {
return nil, fmt.Errorf("docs: open database: %w", err)
@@ -371,6 +377,18 @@ func New(cfg Config) (*App, error) {
return nil, fmt.Errorf("docs: apply migrations: %w", err)
}
+ state, err := readContentSealing(context.Background(), d.G, cipher)
+ if err == nil && state != nil {
+ err = requireSealedContent(context.Background(), d.G)
+ }
+ if err != nil {
+ d.Close()
+ return nil, err
+ }
+ if state == nil {
+ logger.Error("docs: content sealing not activated", "content_encoding", 0, "key_fingerprint", cipher.fingerprint)
+ }
+
// One Runtime for the process (core.New compiles the wasm module
// once — expensive; instantiating a per-document Handle from it is
// not), shared by every document's Store.replay.
@@ -380,6 +398,9 @@ func New(cfg Config) (*App, error) {
return nil, fmt.Errorf("docs: start core runtime: %w", err)
}
store := NewStore(d.G, rt, parseSampleDocuments(cfg.SampleDocuments), logger)
+ if state != nil {
+ store.content = cipher
+ }
// A grant admits its holder. Wired here rather than inside
// resolveAllowlist because the allowlist is resolved before the store
diff --git a/internal/docs/content_cipher.go b/internal/docs/content_cipher.go
new file mode 100644
index 0000000..fc837cc
--- /dev/null
+++ b/internal/docs/content_cipher.go
@@ -0,0 +1,95 @@
+package docs
+
+import (
+ "crypto/aes"
+ "crypto/cipher"
+ "crypto/rand"
+ "crypto/sha256"
+ "encoding/base64"
+ "encoding/binary"
+ "encoding/hex"
+ "errors"
+ "strings"
+
+ "github.com/google/uuid"
+)
+
+const contentEncoding = 1
+const contentUpdateKind byte = 1
+const contentSnapshotKind byte = 2
+const contentKeyCheckKind byte = 3
+
+var developmentContentKey = sha256.Sum256([]byte("docs-development-content-key-not-a-secret"))
+var errContentKey = errors.New("docs: invalid content key")
+var errContentAuthentication = errors.New("docs: content authentication failed")
+var errContentEncoding = errors.New("docs: unsupported content encoding")
+
+type contentCipher struct {
+ aead cipher.AEAD
+ fingerprint string
+}
+
+func newContentCipher(encoded, env string) (*contentCipher, error) {
+ encoded = strings.TrimSpace(encoded)
+ var key []byte
+ if encoded == "" && env == "development" {
+ key = developmentContentKey[:]
+ } else {
+ var err error
+ key, err = base64.StdEncoding.Strict().DecodeString(encoded)
+ if err != nil || len(key) != 32 {
+ return nil, errContentKey
+ }
+ }
+ fingerprint := sha256.Sum256(key)
+ devFingerprint := sha256.Sum256(developmentContentKey[:])
+ if env != "development" && fingerprint == devFingerprint {
+ return nil, errContentKey
+ }
+ block, err := aes.NewCipher(key)
+ if err != nil {
+ return nil, errContentKey
+ }
+ aead, err := cipher.NewGCM(block)
+ if err != nil {
+ return nil, err
+ }
+ return &contentCipher{aead: aead, fingerprint: hex.EncodeToString(fingerprint[:])}, nil
+}
+
+func contentAAD(kind byte, guid string, seq int64, abi uint32) ([]byte, error) {
+ id, err := uuid.Parse(guid)
+ if err != nil || seq < 0 {
+ return nil, errContentAuthentication
+ }
+ aad := append([]byte("oficina/docs/content/v1"), kind)
+ aad = append(aad, id[:]...)
+ aad = binary.BigEndian.AppendUint64(aad, uint64(seq))
+ aad = binary.BigEndian.AppendUint32(aad, abi)
+ return aad, nil
+}
+
+func (c *contentCipher) seal(kind byte, guid string, seq int64, abi uint32, plaintext []byte) ([]byte, error) {
+ aad, err := contentAAD(kind, guid, seq, abi)
+ if err != nil {
+ return nil, err
+ }
+ nonce := make([]byte, c.aead.NonceSize())
+ if _, err := rand.Read(nonce); err != nil {
+ return nil, err
+ }
+ return c.aead.Seal(nonce, nonce, plaintext, aad), nil
+}
+
+func (c *contentCipher) open(kind byte, guid string, seq int64, abi uint32, sealed []byte) ([]byte, error) {
+ aad, err := contentAAD(kind, guid, seq, abi)
+ if err != nil || len(sealed) < c.aead.NonceSize()+c.aead.Overhead() {
+ return nil, errContentAuthentication
+ }
+ n := c.aead.NonceSize()
+ plaintext, err := c.aead.Open(nil, sealed[:n], sealed[n:], aad)
+ if err != nil {
+ return nil, errContentAuthentication
+ }
+ return plaintext, nil
+}
diff --git a/internal/docs/content_cipher_test.go b/internal/docs/content_cipher_test.go
new file mode 100644
index 0000000..d0578b8
--- /dev/null
+++ b/internal/docs/content_cipher_test.go
@@ -0,0 +1,92 @@
+package docs
+
+import (
+ "bytes"
+ "encoding/base64"
+ "errors"
+ "testing"
+
+ "github.com/google/uuid"
+)
+
+func TestContentCipherAuthenticatesRowIdentity(t *testing.T) {
+ c, err := newContentCipher(base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{41}, 32)), "production")
+ if err != nil {
+ t.Fatal(err)
+ }
+ guid := uuid.NewString()
+ body := []byte("private body not shared with another document")
+ sealed, err := c.seal(contentSnapshotKind, guid, 42, 4, body)
+ if err != nil {
+ t.Fatal(err)
+ }
+ got, err := c.open(contentSnapshotKind, guid, 42, 4, sealed)
+ if err != nil || !bytes.Equal(got, body) {
+ t.Fatalf("round trip: %q, %v", got, err)
+ }
+ if bytes.Contains(sealed, body) {
+ t.Fatal("ciphertext contains plaintext")
+ }
+ for _, tc := range []struct {
+ name string
+ kind byte
+ guid string
+ seq int64
+ abi uint32
+ }{
+ {"document", contentSnapshotKind, uuid.NewString(), 42, 4},
+ {"sequence", contentSnapshotKind, guid, 43, 4},
+ {"kind", contentUpdateKind, guid, 42, 4},
+ {"abi", contentSnapshotKind, guid, 42, 5},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ if _, err := c.open(tc.kind, tc.guid, tc.seq, tc.abi, sealed); !errors.Is(err, errContentAuthentication) {
+ t.Fatalf("authentication: %v", err)
+ }
+ })
+ }
+ sealed[len(sealed)-1] ^= 1
+ if _, err := c.open(contentSnapshotKind, guid, 42, 4, sealed); !errors.Is(err, errContentAuthentication) {
+ t.Fatalf("tamper: %v", err)
+ }
+ for n := 0; n < c.aead.NonceSize()+c.aead.Overhead(); n++ {
+ if _, err := c.open(contentSnapshotKind, guid, 42, 4, sealed[:n]); !errors.Is(err, errContentAuthentication) {
+ t.Fatalf("truncation %d: %v", n, err)
+ }
+ }
+}
+
+func TestContentCipherKeysAndNonces(t *testing.T) {
+ for _, encoded := range []string{"", "not-base64", base64.StdEncoding.EncodeToString([]byte("short")), base64.StdEncoding.EncodeToString(developmentContentKey[:])} {
+ if _, err := newContentCipher(encoded, "production"); !errors.Is(err, errContentKey) {
+ t.Fatalf("invalid production key accepted: %v", err)
+ }
+ }
+ a, err := newContentCipher("", "development")
+ if err != nil {
+ t.Fatal(err)
+ }
+ b, err := newContentCipher(base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{42}, 32)), "development")
+ if err != nil {
+ t.Fatal(err)
+ }
+ guid := uuid.NewString()
+ seen := make(map[string]bool)
+ for i := 0; i < 100; i++ {
+ sealed, err := a.seal(contentUpdateKind, guid, 1, 0, []byte("body"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ nonce := string(sealed[:a.aead.NonceSize()])
+ if seen[nonce] {
+ t.Fatal("nonce repeated")
+ }
+ seen[nonce] = true
+ if _, err := b.open(contentUpdateKind, guid, 1, 0, sealed); !errors.Is(err, errContentAuthentication) {
+ t.Fatalf("wrong key: %v", err)
+ }
+ }
+ if _, err := a.seal(contentUpdateKind, "broken-guid", 1, 0, nil); !errors.Is(err, errContentAuthentication) {
+ t.Fatalf("invalid guid: %v", err)
+ }
+}
diff --git a/internal/docs/content_maintenance.go b/internal/docs/content_maintenance.go
new file mode 100644
index 0000000..af3f84d
--- /dev/null
+++ b/internal/docs/content_maintenance.go
@@ -0,0 +1,52 @@
+package docs
+
+import (
+ "context"
+ "errors"
+ "log/slog"
+ "os"
+
+ "github.com/carlosframework/rastrillo/db"
+ "github.com/carlosframework/rastrillo/migrate"
+)
+
+type ContentMaintenanceConfig struct {
+ DBPath string
+ ContentKey string
+ ExpectedFingerprint string
+ Scrub bool
+ Logger *slog.Logger
+}
+
+func MaintainContent(ctx context.Context, cfg ContentMaintenanceConfig) error {
+ c, err := newContentCipher(cfg.ContentKey, "production")
+ if err != nil {
+ return err
+ }
+ if cfg.ExpectedFingerprint == "" || cfg.ExpectedFingerprint != c.fingerprint {
+ return errContentKey
+ }
+ info, err := os.Stat(cfg.DBPath)
+ if err != nil {
+ return err
+ }
+ if !info.Mode().IsRegular() {
+ return errContentState
+ }
+ d, err := db.Open(cfg.DBPath, cfg.Logger)
+ if err != nil {
+ return err
+ }
+ if _, err := migrate.Apply(ctx, d, BootSchema); err != nil {
+ return errors.Join(err, d.Close())
+ }
+ if err := convertContent(ctx, d.G, c); err != nil {
+ return errors.Join(err, d.Close())
+ }
+ if cfg.Scrub {
+ if err := scrubContent(ctx, d, c); err != nil {
+ return errors.Join(err, d.Close())
+ }
+ }
+ return d.Close()
+}
diff --git a/internal/docs/content_maintenance_test.go b/internal/docs/content_maintenance_test.go
new file mode 100644
index 0000000..bf52cac
--- /dev/null
+++ b/internal/docs/content_maintenance_test.go
@@ -0,0 +1,96 @@
+package docs
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+
+ "github.com/carlosframework/rastrillo/db"
+)
+
+func TestContentMaintenanceRequiresExpectedKeyBeforeOpeningDatabase(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "missing.db")
+ encoded := base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{62}, 32))
+ c, err := newContentCipher(encoded, "production")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, fingerprint := range []string{"", "incorrect"} {
+ err := MaintainContent(context.Background(), ContentMaintenanceConfig{DBPath: path, ContentKey: encoded, ExpectedFingerprint: fingerprint})
+ if !errors.Is(err, errContentKey) {
+ t.Fatalf("fingerprint %q: %v", fingerprint, err)
+ }
+ }
+ err = MaintainContent(context.Background(), ContentMaintenanceConfig{DBPath: path, ContentKey: encoded, ExpectedFingerprint: c.fingerprint})
+ if !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("missing database: %v", err)
+ }
+ if _, err := os.Stat(path); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("maintenance created missing database: %v", err)
+ }
+}
+
+func TestContentMaintenancePreservesContentInRestoredCopy(t *testing.T) {
+ ctx := context.Background()
+ path := filepath.Join(t.TempDir(), "source.db")
+ app, err := New(Config{Env: "development", DBPath: path})
+ if err != nil {
+ t.Fatal(err)
+ }
+ doc, err := app.Store().Create(ctx, app.TestPerson(t, "owner@example.com"), "Restore")
+ if err != nil {
+ app.Close()
+ t.Fatal(err)
+ }
+ if _, err := app.Store().ApplyUpdate(ctx, doc, app.TestActorFor(t, doc), validUpdate(t)); err != nil {
+ app.Close()
+ t.Fatal(err)
+ }
+ if err := app.Close(); err != nil {
+ t.Fatal(err)
+ }
+ encoded := base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{63}, 32))
+ c, err := newContentCipher(encoded, "production")
+ if err != nil {
+ t.Fatal(err)
+ }
+ cfg := ContentMaintenanceConfig{DBPath: path, ContentKey: encoded, ExpectedFingerprint: c.fingerprint, Scrub: true}
+ if err := MaintainContent(ctx, cfg); err != nil {
+ t.Fatal(err)
+ }
+ snapshot, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ restoredPath := filepath.Join(t.TempDir(), "restored.db")
+ if err := os.WriteFile(restoredPath, snapshot, 0600); err != nil {
+ t.Fatal(err)
+ }
+ cfg.DBPath = restoredPath
+ if err := MaintainContent(ctx, cfg); err != nil {
+ t.Fatalf("restored maintenance: %v", err)
+ }
+ d, err := db.Open(restoredPath, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer d.Close()
+ state, err := readContentSealing(ctx, d.G, c)
+ if err != nil || state == nil || !state.Scrubbed {
+ t.Fatalf("restored state: %#v, %v", state, err)
+ }
+ var row DocUpdate
+ if err := d.G.Where("doc_id = ?", doc.ID).First(&row).Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := c.openUpdate(doc.GUID, &row); err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(row.UpdateBin, validUpdate(t)) {
+ t.Fatal("restored document changed")
+ }
+}
diff --git a/internal/docs/content_migration.go b/internal/docs/content_migration.go
new file mode 100644
index 0000000..7b82759
--- /dev/null
+++ b/internal/docs/content_migration.go
@@ -0,0 +1,147 @@
+package docs
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+
+ "github.com/google/uuid"
+ "gorm.io/gorm"
+)
+
+var errContentState = errors.New("docs: invalid content sealing state")
+var contentKeyCheck = []byte("oficina/docs/content-key-check/v1")
+
+type contentSealing struct {
+ ID int
+ Encoding int
+ KeyFingerprint string
+ KeyCheck []byte
+ Scrubbed bool
+}
+
+func (contentSealing) TableName() string { return "content_sealing" }
+
+func readContentSealing(ctx context.Context, g *gorm.DB, c *contentCipher) (*contentSealing, error) {
+ var state contentSealing
+ err := g.WithContext(ctx).First(&state, 1).Error
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ var count int64
+ if err := g.WithContext(ctx).Raw("SELECT (SELECT COUNT(*) FROM doc_updates WHERE content_encoding != 0) + (SELECT COUNT(*) FROM doc_snapshots WHERE content_encoding != 0)").Scan(&count).Error; err != nil {
+ return nil, err
+ }
+ if count != 0 {
+ return nil, errContentState
+ }
+ return nil, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ if c == nil || state.KeyFingerprint != c.fingerprint {
+ return nil, errContentKeyMismatch
+ }
+ if state.Encoding != contentEncoding {
+ return nil, errContentState
+ }
+ plain, err := c.open(contentKeyCheckKind, uuid.Nil.String(), 0, uint32(state.Encoding), state.KeyCheck)
+ if err != nil || !bytes.Equal(plain, contentKeyCheck) {
+ return nil, errContentAuthentication
+ }
+ return &state, nil
+}
+
+func convertContent(ctx context.Context, g *gorm.DB, c *contentCipher) error {
+ if c == nil {
+ return errContentKey
+ }
+ return g.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ if err := tx.Exec("PRAGMA secure_delete=ON").Error; err != nil {
+ return err
+ }
+ state, err := readContentSealing(ctx, tx, c)
+ if err != nil {
+ return err
+ }
+ var hadLegacy bool
+ if state != nil {
+ if err := tx.Raw("SELECT EXISTS(SELECT 1 FROM doc_updates WHERE content_encoding = 0 LIMIT 1) OR EXISTS(SELECT 1 FROM doc_snapshots WHERE content_encoding = 0 LIMIT 1)").Scan(&hadLegacy).Error; err != nil {
+ return err
+ }
+ }
+ for _, snapshot := range []bool{false, true} {
+ if err := convertContentRows(ctx, tx, c, snapshot, state != nil); err != nil {
+ return err
+ }
+ }
+ if state != nil {
+ if hadLegacy {
+ return tx.Model(&contentSealing{}).Where("id = 1").Update("scrubbed", false).Error
+ }
+ return nil
+ }
+ check, err := c.seal(contentKeyCheckKind, uuid.Nil.String(), 0, contentEncoding, contentKeyCheck)
+ if err != nil {
+ return err
+ }
+ return tx.Create(&contentSealing{ID: 1, Encoding: contentEncoding, KeyFingerprint: c.fingerprint, KeyCheck: check}).Error
+ })
+}
+
+type contentMigrationRow struct {
+ ID int64
+ DocID int64
+ Seq int64
+ ABIVersion uint32
+ ContentEncoding int
+ Body []byte
+ GUID string
+}
+
+func convertContentRows(ctx context.Context, tx *gorm.DB, c *contentCipher, snapshot, active bool) error {
+ table, column, abi, kind := "doc_updates", "update_bin", "0", contentUpdateKind
+ if snapshot {
+ table, column, abi, kind = "doc_snapshots", "state", "r.abi_version", contentSnapshotKind
+ }
+ last := int64(0)
+ first := true
+ for {
+ var rows []contentMigrationRow
+ query := "SELECT r.id, r.doc_id, r.seq, " + abi + " AS abi_version, r.content_encoding, r." + column + " AS body, d.guid FROM " + table + " r LEFT JOIN documents d ON d.id=r.doc_id WHERE (? OR r.id > ?) ORDER BY r.id LIMIT 100"
+ if err := tx.WithContext(ctx).Raw(query, first, last).Scan(&rows).Error; err != nil {
+ return err
+ }
+ if len(rows) == 0 {
+ return nil
+ }
+ first = false
+ for _, row := range rows {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ last = row.ID
+ if active && row.ContentEncoding == contentEncoding {
+ if _, err := c.open(kind, row.GUID, row.Seq, row.ABIVersion, row.Body); err != nil {
+ return err
+ }
+ continue
+ }
+ if row.ContentEncoding != 0 {
+ return errContentEncoding
+ }
+ sealed, err := c.seal(kind, row.GUID, row.Seq, row.ABIVersion, row.Body)
+ if err != nil {
+ return err
+ }
+ result := tx.Table(table).Where("id = ? AND content_encoding = 0", row.ID).
+ Updates(map[string]any{column: sealed, "content_encoding": contentEncoding})
+ if result.Error != nil {
+ return result.Error
+ }
+ if result.RowsAffected != 1 {
+ return fmt.Errorf("%w: content row %d", errContentState, row.ID)
+ }
+ }
+ }
+}
diff --git a/internal/docs/content_migration_test.go b/internal/docs/content_migration_test.go
new file mode 100644
index 0000000..aac080f
--- /dev/null
+++ b/internal/docs/content_migration_test.go
@@ -0,0 +1,169 @@
+package docs
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+)
+
+func contentMigrationFixture(t *testing.T) (*App, *Document, *contentCipher) {
+ t.Helper()
+ app, err := NewForTest(t)
+ if err != nil {
+ t.Fatal(err)
+ }
+ doc, err := app.Store().Create(context.Background(), app.TestPerson(t, "owner@example.com"), "Migration")
+ if err != nil {
+ t.Fatal(err)
+ }
+ c, err := newContentCipher(base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{53}, 32)), "production")
+ if err != nil {
+ t.Fatal(err)
+ }
+ return app, doc, c
+}
+
+func TestContentMigrationPreservesBodiesAndIsIdempotent(t *testing.T) {
+ app, doc, c := contentMigrationFixture(t)
+ ctx := context.Background()
+ update := DocUpdate{DocID: doc.ID, Seq: 1, UpdateBin: validUpdate(t), CreatedAt: time.Now().UTC()}
+ snapshot := DocSnapshot{DocID: doc.ID, Seq: 1, State: validUpdate(t), ABIVersion: coreABIVersion, CreatedAt: time.Now().UTC()}
+ if err := app.db.G.Create(&update).Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := app.db.G.Create(&snapshot).Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := convertContent(ctx, app.db.G, c); err != nil {
+ t.Fatal(err)
+ }
+ state, err := readContentSealing(ctx, app.db.G, c)
+ if err != nil || state == nil || state.Scrubbed {
+ t.Fatalf("state: %#v, %v", state, err)
+ }
+ if err := app.db.G.First(&update, update.ID).Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := app.db.G.First(&snapshot, snapshot.ID).Error; err != nil {
+ t.Fatal(err)
+ }
+ sealed := append([]byte(nil), update.UpdateBin...)
+ if bytes.Equal(update.UpdateBin, validUpdate(t)) || bytes.Equal(snapshot.State, validUpdate(t)) {
+ t.Fatal("migration left plaintext")
+ }
+ if err := c.openUpdate(doc.GUID, &update); err != nil {
+ t.Fatal(err)
+ }
+ if err := c.openSnapshot(doc.GUID, &snapshot); err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(update.UpdateBin, validUpdate(t)) || !bytes.Equal(snapshot.State, validUpdate(t)) {
+ t.Fatal("migration changed content")
+ }
+ if err := convertContent(ctx, app.db.G, c); err != nil {
+ t.Fatal(err)
+ }
+ if err := app.db.G.First(&update, update.ID).Error; err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(update.UpdateBin, sealed) {
+ t.Fatal("repeat migration rewrote encrypted content")
+ }
+ wrong, _ := newContentCipher("", "development")
+ if _, err := readContentSealing(ctx, app.db.G, wrong); err == nil {
+ t.Fatal("wrong key accepted")
+ }
+}
+
+func TestContentMigrationRollsBackOnInvalidDocument(t *testing.T) {
+ app, doc, c := contentMigrationFixture(t)
+ rows := []DocUpdate{
+ {DocID: doc.ID, Seq: 1, UpdateBin: []byte("valid first row"), CreatedAt: time.Now().UTC()},
+ {DocID: doc.ID + 999, Seq: 1, UpdateBin: []byte("orphan second row"), CreatedAt: time.Now().UTC()},
+ }
+ if err := app.db.G.Create(&rows).Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := convertContent(context.Background(), app.db.G, c); err == nil {
+ t.Fatal("orphan was accepted")
+ }
+ var got DocUpdate
+ if err := app.db.G.First(&got, rows[0].ID).Error; err != nil {
+ t.Fatal(err)
+ }
+ if got.ContentEncoding != 0 || !bytes.Equal(got.UpdateBin, rows[0].UpdateBin) {
+ t.Fatal("failed migration changed first row")
+ }
+ var count int64
+ if err := app.db.G.Model(&contentSealing{}).Count(&count).Error; err != nil {
+ t.Fatal(err)
+ }
+ if count != 0 {
+ t.Fatal("failed migration activated sealing")
+ }
+}
+
+func TestContentRowsRefusePlaintextDowngrade(t *testing.T) {
+ c, err := newContentCipher("", "development")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, encoding := range []int{0, 2, -1} {
+ if err := c.openUpdate("", &DocUpdate{ContentEncoding: encoding, UpdateBin: []byte("plaintext")}); !errors.Is(err, errContentEncoding) {
+ t.Fatalf("update encoding %d: %v", encoding, err)
+ }
+ if err := c.openSnapshot("", &DocSnapshot{ContentEncoding: encoding, State: []byte("plaintext")}); !errors.Is(err, errContentEncoding) {
+ t.Fatalf("snapshot encoding %d: %v", encoding, err)
+ }
+ }
+}
+
+func TestContentScrubRemovesLegacyBytesFromDatabaseAndWAL(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "docs.db")
+ app, err := NewForTestWith(t, Config{DBPath: path})
+ if err != nil {
+ t.Fatal(err)
+ }
+ doc, err := app.Store().Create(context.Background(), app.TestPerson(t, "owner@example.com"), "Scrub")
+ if err != nil {
+ t.Fatal(err)
+ }
+ canary := []byte("CONTENT-SCRUB-CANARY-55de8635-70b3-488a")
+ row := DocUpdate{DocID: doc.ID, Seq: 1, UpdateBin: canary, CreatedAt: time.Now().UTC()}
+ if err := app.db.G.Create(&row).Error; err != nil {
+ t.Fatal(err)
+ }
+ wal, err := os.ReadFile(path + "-wal")
+ if err != nil || !bytes.Contains(wal, canary) {
+ t.Fatalf("legacy WAL canary absent: %v", err)
+ }
+ c, err := newContentCipher("", "development")
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx := context.Background()
+ if err := convertContent(ctx, app.db.G, c); err != nil {
+ t.Fatal(err)
+ }
+ if err := scrubContent(ctx, app.db, c); err != nil {
+ t.Fatal(err)
+ }
+ for _, filename := range []string{path, path + "-wal"} {
+ body, err := os.ReadFile(filename)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if bytes.Contains(body, canary) {
+ t.Fatalf("plaintext retained in %s", filename)
+ }
+ }
+ state, err := readContentSealing(ctx, app.db.G, c)
+ if err != nil || state == nil || !state.Scrubbed {
+ t.Fatalf("scrub state: %#v, %v", state, err)
+ }
+}
diff --git a/internal/docs/content_recovery_test.go b/internal/docs/content_recovery_test.go
new file mode 100644
index 0000000..b87e3f3
--- /dev/null
+++ b/internal/docs/content_recovery_test.go
@@ -0,0 +1,133 @@
+package docs
+
+import (
+ "bytes"
+ "context"
+ "database/sql"
+ "errors"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/rastrillo/db"
+ "gorm.io/gorm"
+)
+
+func TestContentScrubRefusesPinnedWALReader(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "docs.db")
+ app, err := NewForTestWith(t, Config{DBPath: path})
+ if err != nil {
+ t.Fatal(err)
+ }
+ c, err := newContentCipher("", "development")
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx := context.Background()
+ if err := convertContent(ctx, app.db.G, c); err != nil {
+ t.Fatal(err)
+ }
+ if err := scrubContent(ctx, app.db, c); err != nil {
+ t.Fatal(err)
+ }
+ reader, err := sql.Open("sqlite", path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer reader.Close()
+ tx, err := reader.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer tx.Rollback()
+ var scrubbed int
+ if err := tx.QueryRow("SELECT scrubbed FROM content_sealing WHERE id=1").Scan(&scrubbed); err != nil {
+ t.Fatal(err)
+ }
+ if scrubbed != 1 {
+ t.Fatal("reader did not pin scrubbed state")
+ }
+ if _, err := app.db.Writer().Exec("PRAGMA busy_timeout=1"); err != nil {
+ t.Fatal(err)
+ }
+ err = scrubContent(ctx, app.db, c)
+ if err == nil || !strings.Contains(err.Error(), "checkpoint incomplete") {
+ t.Fatalf("pinned reader: %v", err)
+ }
+ state, err := readContentSealing(ctx, app.db.G, c)
+ if err != nil || state == nil || state.Scrubbed {
+ t.Fatalf("failed scrub marker: %#v, %v", state, err)
+ }
+}
+
+func TestContentMigrationRecoversAfterProcessExit(t *testing.T) {
+ ctx := context.Background()
+ c, err := newContentCipher("", "development")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if path := os.Getenv("DOCS_SEAL_CRASH_FIXTURE"); path != "" {
+ d, err := db.Open(path, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer d.Close()
+ if err := d.G.Callback().Update().After("gorm:commit_or_rollback_transaction").Register("docs:test-crash", func(tx *gorm.DB) {
+ if tx.Statement.Table == "doc_updates" && tx.Error == nil && tx.RowsAffected == 1 {
+ os.Exit(73)
+ }
+ }); err != nil {
+ t.Fatal(err)
+ }
+ if err := convertContent(ctx, d.G, c); err != nil {
+ t.Fatal(err)
+ }
+ t.Fatal("conversion never reached crash point")
+ }
+ path := filepath.Join(t.TempDir(), "docs.db")
+ app, err := NewForTestWith(t, Config{DBPath: path})
+ if err != nil {
+ t.Fatal(err)
+ }
+ doc, err := app.Store().Create(ctx, app.TestPerson(t, "owner@example.com"), "Crash recovery")
+ if err != nil {
+ t.Fatal(err)
+ }
+ row := DocUpdate{DocID: doc.ID, Seq: 1, UpdateBin: validUpdate(t), CreatedAt: time.Now().UTC()}
+ if err := app.db.G.Create(&row).Error; err != nil {
+ t.Fatal(err)
+ }
+ command := exec.Command(os.Args[0], "-test.run=^TestContentMigrationRecoversAfterProcessExit$", "-test.count=1")
+ command.Env = append(os.Environ(), "DOCS_SEAL_CRASH_FIXTURE="+path)
+ output, err := command.CombinedOutput()
+ var exited *exec.ExitError
+ if !errors.As(err, &exited) || exited.ExitCode() != 73 {
+ t.Fatalf("child did not exit at write: %v\n%s", err, output)
+ }
+ var recovered DocUpdate
+ if err := app.db.G.First(&recovered, row.ID).Error; err != nil {
+ t.Fatal(err)
+ }
+ if recovered.ContentEncoding != 0 || !bytes.Equal(recovered.UpdateBin, row.UpdateBin) {
+ t.Fatal("process exit committed a partial conversion")
+ }
+ state, err := readContentSealing(ctx, app.db.G, c)
+ if err != nil || state != nil {
+ t.Fatalf("crash activated sealing: %#v, %v", state, err)
+ }
+ if err := convertContent(ctx, app.db.G, c); err != nil {
+ t.Fatalf("retry: %v", err)
+ }
+ if err := app.db.G.First(&recovered, row.ID).Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := c.openUpdate(doc.GUID, &recovered); err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(recovered.UpdateBin, row.UpdateBin) {
+ t.Fatal("retry lost content")
+ }
+}
diff --git a/internal/docs/content_rows.go b/internal/docs/content_rows.go
new file mode 100644
index 0000000..9076e32
--- /dev/null
+++ b/internal/docs/content_rows.go
@@ -0,0 +1,47 @@
+package docs
+
+import "fmt"
+
+func (c *contentCipher) sealUpdate(guid string, row *DocUpdate) error {
+ sealed, err := c.seal(contentUpdateKind, guid, row.Seq, 0, row.UpdateBin)
+ if err != nil {
+ return fmt.Errorf("docs: seal update %d: %w", row.ID, err)
+ }
+ row.UpdateBin = sealed
+ row.ContentEncoding = contentEncoding
+ return nil
+}
+
+func (c *contentCipher) openUpdate(guid string, row *DocUpdate) error {
+ if row.ContentEncoding != contentEncoding {
+ return errContentEncoding
+ }
+ plain, err := c.open(contentUpdateKind, guid, row.Seq, 0, row.UpdateBin)
+ if err != nil {
+ return fmt.Errorf("docs: open update %d: %w", row.ID, err)
+ }
+ row.UpdateBin = plain
+ return nil
+}
+
+func (c *contentCipher) sealSnapshot(guid string, row *DocSnapshot) error {
+ sealed, err := c.seal(contentSnapshotKind, guid, row.Seq, row.ABIVersion, row.State)
+ if err != nil {
+ return fmt.Errorf("docs: seal snapshot %d: %w", row.ID, err)
+ }
+ row.State = sealed
+ row.ContentEncoding = contentEncoding
+ return nil
+}
+
+func (c *contentCipher) openSnapshot(guid string, row *DocSnapshot) error {
+ if row.ContentEncoding != contentEncoding {
+ return errContentEncoding
+ }
+ plain, err := c.open(contentSnapshotKind, guid, row.Seq, row.ABIVersion, row.State)
+ if err != nil {
+ return fmt.Errorf("docs: open snapshot %d: %w", row.ID, err)
+ }
+ row.State = plain
+ return nil
+}
diff --git a/internal/docs/content_runtime.go b/internal/docs/content_runtime.go
new file mode 100644
index 0000000..ce5d239
--- /dev/null
+++ b/internal/docs/content_runtime.go
@@ -0,0 +1,67 @@
+package docs
+
+import (
+ "context"
+ "errors"
+
+ "gorm.io/gorm"
+)
+
+var errContentKeyMismatch = errors.New("docs: content key does not match this database")
+var errContentMixed = errors.New("docs: mixed content encodings; run docs-seal offline")
+
+func requireSealedContent(ctx context.Context, g *gorm.DB) error {
+ var mixed bool
+ if err := g.WithContext(ctx).Raw(`SELECT EXISTS(SELECT 1 FROM doc_updates WHERE content_encoding != 1 LIMIT 1) OR EXISTS(SELECT 1 FROM doc_snapshots WHERE content_encoding != 1 LIMIT 1)`).Scan(&mixed).Error; err != nil {
+ return err
+ }
+ if mixed {
+ return errContentMixed
+ }
+ return nil
+}
+
+func refuseActivatedLegacyWrite(tx *gorm.DB) error {
+ var count int64
+ if err := tx.Model(&contentSealing{}).Where("id = 1").Count(&count).Error; err != nil {
+ return err
+ }
+ if count != 0 {
+ return errContentState
+ }
+ return nil
+}
+
+func (s *Store) prepareContentUpdate(tx *gorm.DB, guid string, row *DocUpdate) error {
+ if s.content == nil {
+ return refuseActivatedLegacyWrite(tx)
+ }
+ return s.content.sealUpdate(guid, row)
+}
+
+func (s *Store) prepareContentSnapshot(tx *gorm.DB, guid string, row *DocSnapshot) error {
+ if s.content == nil {
+ return refuseActivatedLegacyWrite(tx)
+ }
+ return s.content.sealSnapshot(guid, row)
+}
+
+func (s *Store) openContentUpdate(guid string, row *DocUpdate) error {
+ if s.content == nil {
+ if row.ContentEncoding != 0 {
+ return errContentEncoding
+ }
+ return nil
+ }
+ return s.content.openUpdate(guid, row)
+}
+
+func (s *Store) openContentSnapshot(guid string, row *DocSnapshot) error {
+ if s.content == nil {
+ if row.ContentEncoding != 0 {
+ return errContentEncoding
+ }
+ return nil
+ }
+ return s.content.openSnapshot(guid, row)
+}
diff --git a/internal/docs/content_runtime_test.go b/internal/docs/content_runtime_test.go
new file mode 100644
index 0000000..17b081a
--- /dev/null
+++ b/internal/docs/content_runtime_test.go
@@ -0,0 +1,329 @@
+package docs
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "errors"
+ "log/slog"
+ "net/http"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "gorm.io/gorm"
+)
+
+func activatedContentApp(t *testing.T) (*App, string) {
+ t.Helper()
+ key := base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{71}, 32))
+ cfg := Config{DBPath: filepath.Join(t.TempDir(), "docs.db"), Env: "development", ContentKey: key}
+ app, err := New(cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := app.Close(); err != nil {
+ t.Fatal(err)
+ }
+ cipher, err := newContentCipher(key, "production")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := MaintainContent(context.Background(), ContentMaintenanceConfig{DBPath: cfg.DBPath, ContentKey: key, ExpectedFingerprint: cipher.fingerprint}); err != nil {
+ t.Fatal(err)
+ }
+ app, err = New(cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() {
+ if err := app.Close(); err != nil {
+ t.Error(err)
+ }
+ })
+ return app, key
+}
+
+func TestActivatedContentPersistsSealedAndServesPlainUpdates(t *testing.T) {
+ app, _ := activatedContentApp(t)
+ ctx := context.Background()
+ owner := app.TestPerson(t, "owner@example.com")
+ doc, err := app.Store().Create(ctx, owner, "Sealed")
+ if err != nil {
+ t.Fatal(err)
+ }
+ req := app.TestRequest(t, "owner@example.com", http.MethodPost, "/d/"+doc.Sqid+"/updates", bytes.NewReader(validUpdate(t)))
+ if rec := serve(app, req); rec.Code != http.StatusNoContent {
+ t.Fatalf("POST: %d %s", rec.Code, rec.Body.String())
+ }
+ var row DocUpdate
+ if err := app.store.db.First(&row).Error; err != nil {
+ t.Fatal(err)
+ }
+ if row.ContentEncoding != 1 || bytes.Contains(row.UpdateBin, []byte("First paragraph.")) || bytes.Equal(row.UpdateBin, validUpdate(t)) {
+ t.Fatal("persisted content is not sealed")
+ }
+ for _, suffix := range []string{"/updates", "/updates?raw=1"} {
+ rec := serve(app, app.TestRequest(t, "owner@example.com", http.MethodGet, "/d/"+doc.Sqid+suffix, nil))
+ if rec.Code != 200 || !bytes.Equal(rec.Body.Bytes(), validUpdate(t)) {
+ t.Fatalf("GET %s: %d", suffix, rec.Code)
+ }
+ }
+ h, seq, _, err := app.store.replay(ctx, doc)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer h.Close(ctx)
+ if err := app.store.writeSnapshot(ctx, doc, h, seq); err != nil {
+ t.Fatal(err)
+ }
+ var snap DocSnapshot
+ if err := app.store.db.First(&snap).Error; err != nil {
+ t.Fatal(err)
+ }
+ if snap.ContentEncoding != 1 || bytes.Contains(snap.State, []byte("First paragraph.")) {
+ t.Fatal("snapshot is not sealed")
+ }
+ rec := serve(app, app.TestRequest(t, "owner@example.com", http.MethodGet, "/d/"+doc.Sqid+"/updates", nil))
+ if rec.Code != 200 || bytes.Equal(rec.Body.Bytes(), snap.State) {
+ t.Fatalf("snapshot GET: %d", rec.Code)
+ }
+ if text, err := app.store.Text(ctx, doc); err != nil || !strings.Contains(text, "First paragraph.") {
+ t.Fatalf("sealed replay text: %q %v", text, err)
+ }
+ replayed, _, _, err := app.store.replay(ctx, doc)
+ if err != nil {
+ t.Fatal(err)
+ }
+ replayed.Close(ctx)
+ row.UpdateBin[len(row.UpdateBin)-1] ^= 1
+ if err := app.store.db.Model(&row).Update("update_bin", row.UpdateBin).Error; err != nil {
+ t.Fatal(err)
+ }
+ rec = serve(app, app.TestRequest(t, "owner@example.com", http.MethodGet, "/d/"+doc.Sqid+"/updates?raw=1", nil))
+ if rec.Code != 500 || strings.Contains(rec.Body.String(), "First paragraph.") || rec.Header().Get("Docs-Seq") != "" {
+ t.Fatal("corrupt update was exposed or acknowledged")
+ }
+}
+
+func TestLegacyContentWriterRefusesAfterActivation(t *testing.T) {
+ app, err := NewForTest(t)
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx := context.Background()
+ owner := app.TestPerson(t, "owner@example.com")
+ doc, err := app.store.Create(ctx, owner, "Legacy")
+ if err != nil {
+ t.Fatal(err)
+ }
+ h, err := app.core.NewDoc(ctx, doc.GUID, doc.ClientID)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer h.Close(ctx)
+ if err := h.Apply(ctx, validUpdate(t)); err != nil {
+ t.Fatal(err)
+ }
+ cipher, err := newContentCipher("", "development")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := convertContent(ctx, app.store.db, cipher); err != nil {
+ t.Fatal(err)
+ }
+ if seq, err := app.store.ApplyUpdate(ctx, doc, Actor{PersonID: owner, Subject: "owner@example.com"}, validUpdate(t)); !errors.Is(err, errContentState) || seq != 0 {
+ t.Fatalf("stale Store.ApplyUpdate committed: seq %d error %v", seq, err)
+ }
+ update := &DocUpdate{DocID: doc.ID, Seq: 1, UpdateBin: validUpdate(t), CreatedAt: time.Now().UTC()}
+ err = app.store.db.Transaction(func(tx *gorm.DB) error {
+ if err := app.store.prepareContentUpdate(tx, doc.GUID, update); err != nil {
+ return err
+ }
+ return tx.Create(update).Error
+ })
+ if !errors.Is(err, errContentState) {
+ t.Fatalf("legacy update: %v", err)
+ }
+ if err := app.store.writeSnapshot(ctx, doc, h, 1); !errors.Is(err, errContentState) {
+ t.Fatalf("legacy snapshot: %v", err)
+ }
+ var count int64
+ if err := app.store.db.Model(&DocUpdate{}).Count(&count).Error; err != nil || count != 0 {
+ t.Fatalf("legacy write committed: %d %v", count, err)
+ }
+}
+
+func TestContentStartupRejectsMixedStateAndMaintenanceRepairsWithoutMarkerRewrite(t *testing.T) {
+ app, key := activatedContentApp(t)
+ ctx := context.Background()
+ doc, err := app.store.Create(ctx, app.TestPerson(t, "owner@example.com"), "Mixed")
+ if err != nil {
+ t.Fatal(err)
+ }
+ var before contentSealing
+ if err := app.store.db.First(&before).Error; err != nil {
+ t.Fatal(err)
+ }
+ row := DocUpdate{DocID: doc.ID, Seq: 1, UpdateBin: validUpdate(t), CreatedAt: time.Now().UTC()}
+ if err := app.store.db.Create(&row).Error; err != nil {
+ t.Fatal(err)
+ }
+ candidate, err := New(app.cfg)
+ if candidate != nil {
+ candidate.Close()
+ }
+ if !errors.Is(err, errContentMixed) {
+ t.Fatalf("mixed boot: %v", err)
+ }
+ if err := convertContent(ctx, app.store.db, app.store.content); err != nil {
+ t.Fatal(err)
+ }
+ var after contentSealing
+ if err := app.store.db.First(&after).Error; err != nil {
+ t.Fatal(err)
+ }
+ if before.KeyFingerprint != after.KeyFingerprint || !bytes.Equal(before.KeyCheck, after.KeyCheck) {
+ t.Fatal("repair rewrote activation marker")
+ }
+ if err := app.store.db.First(&row).Error; err != nil {
+ t.Fatal(err)
+ }
+ if row.ContentEncoding != 1 {
+ t.Fatal("repair retained legacy row")
+ }
+ cfg := app.cfg
+ cfg.ContentKey = base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{72}, 32))
+ candidate, err = New(cfg)
+ if candidate != nil {
+ candidate.Close()
+ }
+ if !errors.Is(err, errContentKeyMismatch) {
+ t.Fatalf("wrong-key boot: %v", err)
+ }
+ cfg.ContentKey = key
+ candidate, err = New(cfg)
+ if err != nil {
+ t.Fatal(err)
+ }
+ candidate.Close()
+ if err := app.store.db.Delete(&contentSealing{}, 1).Error; err != nil {
+ t.Fatal(err)
+ }
+ candidate, err = New(cfg)
+ if candidate != nil {
+ candidate.Close()
+ }
+ if !errors.Is(err, errContentState) {
+ t.Fatalf("missing-marker boot: %v", err)
+ }
+}
+
+func TestLegacyBootDiagnosesUnactivatedContent(t *testing.T) {
+ var logs bytes.Buffer
+ app, err := NewForTestWith(t, Config{Logger: slog.New(slog.NewJSONHandler(&logs, nil))})
+ if err != nil {
+ t.Fatal(err)
+ }
+ if app.store.content != nil {
+ t.Fatal("legacy boot activated content")
+ }
+ if !strings.Contains(logs.String(), `"level":"ERROR"`) || !strings.Contains(logs.String(), `"content_encoding":0`) {
+ t.Fatalf("legacy diagnosis absent: %s", logs.String())
+ }
+}
+
+func TestContentRepairFailurePreservesActivatedMarkerAndLegacyRows(t *testing.T) {
+ app, _ := activatedContentApp(t)
+ doc, err := app.store.Create(context.Background(), app.TestPerson(t, "owner@example.com"), "Repair")
+ if err != nil {
+ t.Fatal(err)
+ }
+ var before contentSealing
+ if err := app.store.db.First(&before).Error; err != nil {
+ t.Fatal(err)
+ }
+ rows := []DocUpdate{{DocID: doc.ID, Seq: 1, UpdateBin: validUpdate(t), CreatedAt: time.Now().UTC()}, {DocID: doc.ID, Seq: 2, UpdateBin: validUpdate(t), ContentEncoding: 99, CreatedAt: time.Now().UTC()}}
+ if err := app.store.db.Create(&rows).Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := convertContent(context.Background(), app.store.db, app.store.content); !errors.Is(err, errContentEncoding) {
+ t.Fatalf("repair failure: %v", err)
+ }
+ var after contentSealing
+ if err := app.store.db.First(&after).Error; err != nil {
+ t.Fatal(err)
+ }
+ if before.Encoding != after.Encoding || before.KeyFingerprint != after.KeyFingerprint || !bytes.Equal(before.KeyCheck, after.KeyCheck) || before.Scrubbed != after.Scrubbed {
+ t.Fatal("failed repair changed marker")
+ }
+ var row DocUpdate
+ if err := app.store.db.First(&row, rows[0].ID).Error; err != nil {
+ t.Fatal(err)
+ }
+ if row.ContentEncoding != 0 || !bytes.Equal(row.UpdateBin, validUpdate(t)) {
+ t.Fatal("failed repair partially converted legacy row")
+ }
+}
+
+func TestProductionBootRequiresContentKeyBeforeOpeningLegacyDatabase(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "docs.db")
+ cfg := Config{DBPath: path, Env: "production", Origin: "https://docs.example.com", InstanceKey: "independent auth key", Mailer: &recordingMailer{}, AllowedAddresses: "owner@example.com"}
+ app, err := New(cfg)
+ if app != nil {
+ app.Close()
+ }
+ if !errors.Is(err, errContentKey) {
+ t.Fatalf("missing production content key: %v", err)
+ }
+}
+
+func TestStaleLegacyStoreRefusesSealedUpdatesAndSnapshots(t *testing.T) {
+ app, _ := activatedContentApp(t)
+ ctx := context.Background()
+ owner := app.TestPerson(t, "owner@example.com")
+ doc, err := app.store.Create(ctx, owner, "Stale")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := app.store.ApplyUpdate(ctx, doc, Actor{PersonID: owner, Subject: "owner@example.com"}, validUpdate(t)); err != nil {
+ t.Fatal(err)
+ }
+ h, seq, _, err := app.store.replay(ctx, doc)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if err := app.store.writeSnapshot(ctx, doc, h, seq); err != nil {
+ h.Close(ctx)
+ t.Fatal(err)
+ }
+ h.Close(ctx)
+ app.store.content = nil
+ if _, err := app.store.newestSnapshot(ctx, doc.ID); !errors.Is(err, errContentEncoding) {
+ t.Fatalf("legacy snapshot read: %v", err)
+ }
+ rec := serve(app, app.TestRequest(t, "owner@example.com", http.MethodGet, "/d/"+doc.Sqid+"/updates?raw=1", nil))
+ if rec.Code != http.StatusInternalServerError || rec.Header().Get("Docs-Seq") != "" {
+ t.Fatalf("legacy update GET: %d, seq %q", rec.Code, rec.Header().Get("Docs-Seq"))
+ }
+}
+
+func TestContentStartupRejectsPlaintextSnapshotWithoutLegacyUpdates(t *testing.T) {
+ app, _ := activatedContentApp(t)
+ doc, err := app.store.Create(context.Background(), app.TestPerson(t, "owner@example.com"), "Snapshot")
+ if err != nil {
+ t.Fatal(err)
+ }
+ snapshot := DocSnapshot{DocID: doc.ID, Seq: 1, State: validUpdate(t), ABIVersion: coreABIVersion, CreatedAt: time.Now().UTC()}
+ if err := app.store.db.Create(&snapshot).Error; err != nil {
+ t.Fatal(err)
+ }
+ candidate, err := New(app.cfg)
+ if candidate != nil {
+ candidate.Close()
+ }
+ if !errors.Is(err, errContentMixed) {
+ t.Fatalf("plaintext snapshot boot: %v", err)
+ }
+}
diff --git a/internal/docs/content_scrub.go b/internal/docs/content_scrub.go
new file mode 100644
index 0000000..be113f7
--- /dev/null
+++ b/internal/docs/content_scrub.go
@@ -0,0 +1,52 @@
+package docs
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+
+ "github.com/carlosframework/rastrillo/db"
+)
+
+func scrubContent(ctx context.Context, d *db.DB, c *contentCipher) error {
+ state, err := readContentSealing(ctx, d.G, c)
+ if err != nil {
+ return err
+ }
+ if state == nil {
+ return errContentState
+ }
+ if err := convertContent(ctx, d.G, c); err != nil {
+ return err
+ }
+ conn, err := d.Writer().Conn(ctx)
+ if err != nil {
+ return err
+ }
+ defer conn.Close()
+ if _, err := conn.ExecContext(ctx, "UPDATE content_sealing SET scrubbed = 0 WHERE id = 1"); err != nil {
+ return err
+ }
+ if err := truncateContentWAL(ctx, conn); err != nil {
+ return err
+ }
+ if _, err := conn.ExecContext(ctx, "VACUUM"); err != nil {
+ return err
+ }
+ if err := truncateContentWAL(ctx, conn); err != nil {
+ return err
+ }
+ _, err = conn.ExecContext(ctx, "UPDATE content_sealing SET scrubbed = 1 WHERE id = 1")
+ return err
+}
+
+func truncateContentWAL(ctx context.Context, conn *sql.Conn) error {
+ var busy, frames, checkpointed int
+ if err := conn.QueryRowContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &frames, &checkpointed); err != nil {
+ return err
+ }
+ if busy != 0 || frames != checkpointed {
+ return fmt.Errorf("docs: checkpoint incomplete (%d, %d, %d)", busy, frames, checkpointed)
+ }
+ return nil
+}
diff --git a/internal/docs/handlers.go b/internal/docs/handlers.go
index 6c14a80..53061c8 100644
--- a/internal/docs/handlers.go
+++ b/internal/docs/handlers.go
@@ -738,6 +738,14 @@ func (h *handlers) updatesGet(w http.ResponseWriter, r *http.Request) {
return
}
+ for i := range updates {
+ if err := h.store.openContentUpdate(doc.GUID, &updates[i]); err != nil {
+ h.logger.Error("docs: load update log", "sqid", doc.Sqid, "err", err)
+ http.Error(w, rastrillo.T(r, "docs.error.generic"), http.StatusInternalServerError)
+ return
+ }
+ }
+
total := 0
if snapshot != nil {
total = len(snapshot.State)
diff --git a/internal/docs/migrations/0007_content_encoding.sql b/internal/docs/migrations/0007_content_encoding.sql
new file mode 100644
index 0000000..8b90a14
--- /dev/null
+++ b/internal/docs/migrations/0007_content_encoding.sql
@@ -0,0 +1,10 @@
+ALTER TABLE doc_updates ADD COLUMN content_encoding INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE doc_snapshots ADD COLUMN content_encoding INTEGER NOT NULL DEFAULT 0;
+
+CREATE TABLE IF NOT EXISTS content_sealing (
+ id INTEGER PRIMARY KEY CHECK (id = 1),
+ encoding INTEGER NOT NULL,
+ key_fingerprint TEXT NOT NULL,
+ key_check BLOB NOT NULL,
+ scrubbed INTEGER NOT NULL DEFAULT 0
+);
diff --git a/internal/docs/models.go b/internal/docs/models.go
index c95c8d6..6758361 100644
--- a/internal/docs/models.go
+++ b/internal/docs/models.go
@@ -78,18 +78,13 @@ type Person struct {
// DocUpdate is one Yjs update in a document's log, applied in Seq
// order to rebuild the document's CRDT state (Store.replay).
-//
-// UpdateBin is plaintext in this table. Sealing it (spec §2 — per-row
-// AES-GCM, same as doc titles and everything else that is document
-// content) is out of scope for this plan; this comment exists so the
-// next reader does not mistake that for a decision that it stays
-// plaintext.
type DocUpdate struct {
ID int64 `gorm:"primaryKey"`
DocID int64 `gorm:"column:doc_id;not null;index"`
Seq int64 `gorm:"column:seq;not null"`
- UpdateBin []byte `gorm:"column:update_bin;not null"`
+ UpdateBin []byte `gorm:"column:update_bin;not null"`
+ ContentEncoding int `gorm:"column:content_encoding;not null"`
CreatedAt time.Time `gorm:"column:created_at;not null"`
}
@@ -100,10 +95,6 @@ type DocUpdate struct {
// a snapshot that cannot be read is discarded in favour of replaying
// them (the snapshots design, D2).
//
-// State is plaintext in this table, exactly as DocUpdate.UpdateBin is.
-// Worth one caveat the sealing work should carry: a doc_updates row is a
-// fragment of an edit, while this is the entire document in one blob.
-//
// ABIVersion records the core that produced State. Without it, "discard
// a snapshot from an older ABI" is unenforceable prose.
type DocSnapshot struct {
@@ -111,8 +102,9 @@ type DocSnapshot struct {
DocID int64 `gorm:"column:doc_id;not null;index"`
Seq int64 `gorm:"column:seq;not null"`
- State []byte `gorm:"column:state;not null"`
- ABIVersion uint32 `gorm:"column:abi_version;not null"`
+ State []byte `gorm:"column:state;not null"`
+ ContentEncoding int `gorm:"column:content_encoding;not null"`
+ ABIVersion uint32 `gorm:"column:abi_version;not null"`
CreatedAt time.Time `gorm:"column:created_at;not null"`
}
diff --git a/internal/docs/snapshots.go b/internal/docs/snapshots.go
index e632f65..3ffe465 100644
--- a/internal/docs/snapshots.go
+++ b/internal/docs/snapshots.go
@@ -49,6 +49,13 @@ func (s *Store) newestSnapshot(ctx context.Context, docID int64) (*DocSnapshot,
if err != nil {
return nil, fmt.Errorf("docs: newest snapshot for document %d: %w", docID, err)
}
+ var doc Document
+ if err := s.db.WithContext(ctx).Select("guid").First(&doc, docID).Error; err != nil {
+ return nil, err
+ }
+ if err := s.openContentSnapshot(doc.GUID, &row); err != nil {
+ return nil, err
+ }
return &row, nil
}
@@ -102,6 +109,9 @@ func (s *Store) writeSnapshot(ctx context.Context, doc *Document, h *core.Handle
CreatedAt: time.Now().UTC(),
}
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ if err := s.prepareContentSnapshot(tx, doc.GUID, row); err != nil {
+ return err
+ }
if err := tx.Create(row).Error; err != nil {
return fmt.Errorf("docs: persist snapshot for document %d at seq %d: %w", doc.ID, seq, err)
}
diff --git a/internal/docs/store.go b/internal/docs/store.go
index 5ef83f9..5abefeb 100644
--- a/internal/docs/store.go
+++ b/internal/docs/store.go
@@ -21,8 +21,9 @@ import (
// the update log, and the document's actual text read back through
// the real CRDT core rather than any column here — see Text.
type Store struct {
- db *gorm.DB
- core *core.Runtime
+ db *gorm.DB
+ core *core.Runtime
+ content *contentCipher
// sampleDocuments mirrors Config.SampleDocuments, already parsed —
// see EnsurePerson's own doc comment for what it gates and why the
@@ -451,6 +452,10 @@ func (s *Store) replay(ctx context.Context, doc *Document) (h *core.Handle, repl
}
for _, u := range updates {
+ if err := s.openContentUpdate(doc.GUID, &u); err != nil {
+ h.Close(ctx)
+ return nil, 0, 0, err
+ }
applyErr := h.Apply(ctx, u.UpdateBin)
if errors.Is(applyErr, core.ErrUpdateLogFull) {
// This handle has retained core.MaxRetainedUpdates updates
@@ -571,10 +576,7 @@ func (s *Store) reseedForReplay(ctx context.Context, doc *Document, full *core.H
}
// Text returns doc's current text — read from the real CRDT core by
-// replaying its update log, never from a database column. Sealing
-// (spec §2 — per-row encryption of update_bin, same as document
-// titles) is out of scope for this plan; this plaintext replay is not
-// a decision that content stays unsealed.
+// replaying its update log, never from a database column.
func (s *Store) Text(ctx context.Context, doc *Document) (string, error) {
h, _, _, err := s.replay(ctx, doc)
if err != nil {
@@ -853,6 +855,9 @@ func (s *Store) ApplyUpdate(ctx context.Context, doc *Document, actor Actor, upd
UpdateBin: update,
CreatedAt: time.Now().UTC(),
}
+ if err := s.prepareContentUpdate(tx, doc.GUID, row); err != nil {
+ return err
+ }
if err := tx.Create(row).Error; err != nil {
return fmt.Errorf("docs: persist update for document %d: %w", doc.ID, err)
}