amadan

oficina / docs Public

Prepare atomic content conversion and explicit database scrub

Paul Campbell pushed by paul@keymail.dev aab3abc2aca6fd007a4c9e55f2f8ba2fe6ccef85 parent fbe7ec6
6 files changed, +421 −3
  • internal/docs/content_migration.go +138 −0
  • internal/docs/content_migration_test.go +169 −0
  • internal/docs/content_rows.go +47 −0
  • internal/docs/content_scrub.go +52 −0
  • internal/docs/migrations/0007_content_encoding.sql +10 −0
  • internal/docs/models.go +5 −3
diff --git a/internal/docs/content_migration.go b/internal/docs/content_migration.go
new file mode 100644
index 0000000..9115fb1
--- /dev/null
+++ b/internal/docs/content_migration.go
@@ -0,0 +1,138 @@
+package docs
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+
+ "github.com/google/uuid"
+ "gorm.io/gorm"
+)
+
+var errContentState = errors.New("docs: invalid content sealing state")
+var contentKeyCheck = []byte("oficina/docs/content-key-check/v1")
+
+type contentSealing struct {
+ ID int
+ Encoding int
+ KeyFingerprint string
+ KeyCheck []byte
+ Scrubbed bool
+}
+
+func (contentSealing) TableName() string { return "content_sealing" }
+
+func readContentSealing(ctx context.Context, g *gorm.DB, c *contentCipher) (*contentSealing, error) {
+ var state contentSealing
+ err := g.WithContext(ctx).First(&state, 1).Error
+ if errors.Is(err, gorm.ErrRecordNotFound) {
+ var count int64
+ if err := g.WithContext(ctx).Raw("SELECT (SELECT COUNT(*) FROM doc_updates WHERE content_encoding != 0) + (SELECT COUNT(*) FROM doc_snapshots WHERE content_encoding != 0)").Scan(&count).Error; err != nil {
+ return nil, err
+ }
+ if count != 0 {
+ return nil, errContentState
+ }
+ return nil, nil
+ }
+ if err != nil {
+ return nil, err
+ }
+ if state.Encoding != contentEncoding || c == nil || state.KeyFingerprint != c.fingerprint {
+ return nil, errContentState
+ }
+ plain, err := c.open(contentKeyCheckKind, uuid.Nil.String(), 0, uint32(state.Encoding), state.KeyCheck)
+ if err != nil || !bytes.Equal(plain, contentKeyCheck) {
+ return nil, errContentAuthentication
+ }
+ return &state, nil
+}
+
+func convertContent(ctx context.Context, g *gorm.DB, c *contentCipher) error {
+ if c == nil {
+ return errContentKey
+ }
+ return g.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
+ if err := tx.Exec("PRAGMA secure_delete=ON").Error; err != nil {
+ return err
+ }
+ state, err := readContentSealing(ctx, tx, c)
+ if err != nil {
+ return err
+ }
+ for _, snapshot := range []bool{false, true} {
+ if err := convertContentRows(ctx, tx, c, snapshot, state != nil); err != nil {
+ return err
+ }
+ }
+ if state != nil {
+ return nil
+ }
+ check, err := c.seal(contentKeyCheckKind, uuid.Nil.String(), 0, contentEncoding, contentKeyCheck)
+ if err != nil {
+ return err
+ }
+ return tx.Create(&contentSealing{ID: 1, Encoding: contentEncoding, KeyFingerprint: c.fingerprint, KeyCheck: check}).Error
+ })
+}
+
+type contentMigrationRow struct {
+ ID int64
+ DocID int64
+ Seq int64
+ ABIVersion uint32
+ ContentEncoding int
+ Body []byte
+ GUID string
+}
+
+func convertContentRows(ctx context.Context, tx *gorm.DB, c *contentCipher, snapshot, active bool) error {
+ table, column, abi, kind := "doc_updates", "update_bin", "0", contentUpdateKind
+ if snapshot {
+ table, column, abi, kind = "doc_snapshots", "state", "r.abi_version", contentSnapshotKind
+ }
+ last := int64(0)
+ first := true
+ for {
+ var rows []contentMigrationRow
+ query := "SELECT r.id, r.doc_id, r.seq, " + abi + " AS abi_version, r.content_encoding, r." + column + " AS body, d.guid FROM " + table + " r LEFT JOIN documents d ON d.id=r.doc_id WHERE (? OR r.id > ?) ORDER BY r.id LIMIT 100"
+ if err := tx.WithContext(ctx).Raw(query, first, last).Scan(&rows).Error; err != nil {
+ return err
+ }
+ if len(rows) == 0 {
+ return nil
+ }
+ first = false
+ for _, row := range rows {
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ last = row.ID
+ if active {
+ if row.ContentEncoding != contentEncoding {
+ return errContentEncoding
+ }
+ if _, err := c.open(kind, row.GUID, row.Seq, row.ABIVersion, row.Body); err != nil {
+ return err
+ }
+ continue
+ }
+ if row.ContentEncoding != 0 {
+ return errContentEncoding
+ }
+ sealed, err := c.seal(kind, row.GUID, row.Seq, row.ABIVersion, row.Body)
+ if err != nil {
+ return err
+ }
+ result := tx.Table(table).Where("id = ? AND content_encoding = 0", row.ID).
+ Updates(map[string]any{column: sealed, "content_encoding": contentEncoding})
+ if result.Error != nil {
+ return result.Error
+ }
+ if result.RowsAffected != 1 {
+ return fmt.Errorf("%w: content row %d", errContentState, row.ID)
+ }
+ }
+ }
+}
diff --git a/internal/docs/content_migration_test.go b/internal/docs/content_migration_test.go
new file mode 100644
index 0000000..aac080f
--- /dev/null
+++ b/internal/docs/content_migration_test.go
@@ -0,0 +1,169 @@
+package docs
+
+import (
+ "bytes"
+ "context"
+ "encoding/base64"
+ "errors"
+ "os"
+ "path/filepath"
+ "testing"
+ "time"
+)
+
+func contentMigrationFixture(t *testing.T) (*App, *Document, *contentCipher) {
+ t.Helper()
+ app, err := NewForTest(t)
+ if err != nil {
+ t.Fatal(err)
+ }
+ doc, err := app.Store().Create(context.Background(), app.TestPerson(t, "owner@example.com"), "Migration")
+ if err != nil {
+ t.Fatal(err)
+ }
+ c, err := newContentCipher(base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{53}, 32)), "production")
+ if err != nil {
+ t.Fatal(err)
+ }
+ return app, doc, c
+}
+
+func TestContentMigrationPreservesBodiesAndIsIdempotent(t *testing.T) {
+ app, doc, c := contentMigrationFixture(t)
+ ctx := context.Background()
+ update := DocUpdate{DocID: doc.ID, Seq: 1, UpdateBin: validUpdate(t), CreatedAt: time.Now().UTC()}
+ snapshot := DocSnapshot{DocID: doc.ID, Seq: 1, State: validUpdate(t), ABIVersion: coreABIVersion, CreatedAt: time.Now().UTC()}
+ if err := app.db.G.Create(&update).Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := app.db.G.Create(&snapshot).Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := convertContent(ctx, app.db.G, c); err != nil {
+ t.Fatal(err)
+ }
+ state, err := readContentSealing(ctx, app.db.G, c)
+ if err != nil || state == nil || state.Scrubbed {
+ t.Fatalf("state: %#v, %v", state, err)
+ }
+ if err := app.db.G.First(&update, update.ID).Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := app.db.G.First(&snapshot, snapshot.ID).Error; err != nil {
+ t.Fatal(err)
+ }
+ sealed := append([]byte(nil), update.UpdateBin...)
+ if bytes.Equal(update.UpdateBin, validUpdate(t)) || bytes.Equal(snapshot.State, validUpdate(t)) {
+ t.Fatal("migration left plaintext")
+ }
+ if err := c.openUpdate(doc.GUID, &update); err != nil {
+ t.Fatal(err)
+ }
+ if err := c.openSnapshot(doc.GUID, &snapshot); err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(update.UpdateBin, validUpdate(t)) || !bytes.Equal(snapshot.State, validUpdate(t)) {
+ t.Fatal("migration changed content")
+ }
+ if err := convertContent(ctx, app.db.G, c); err != nil {
+ t.Fatal(err)
+ }
+ if err := app.db.G.First(&update, update.ID).Error; err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(update.UpdateBin, sealed) {
+ t.Fatal("repeat migration rewrote encrypted content")
+ }
+ wrong, _ := newContentCipher("", "development")
+ if _, err := readContentSealing(ctx, app.db.G, wrong); err == nil {
+ t.Fatal("wrong key accepted")
+ }
+}
+
+func TestContentMigrationRollsBackOnInvalidDocument(t *testing.T) {
+ app, doc, c := contentMigrationFixture(t)
+ rows := []DocUpdate{
+ {DocID: doc.ID, Seq: 1, UpdateBin: []byte("valid first row"), CreatedAt: time.Now().UTC()},
+ {DocID: doc.ID + 999, Seq: 1, UpdateBin: []byte("orphan second row"), CreatedAt: time.Now().UTC()},
+ }
+ if err := app.db.G.Create(&rows).Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := convertContent(context.Background(), app.db.G, c); err == nil {
+ t.Fatal("orphan was accepted")
+ }
+ var got DocUpdate
+ if err := app.db.G.First(&got, rows[0].ID).Error; err != nil {
+ t.Fatal(err)
+ }
+ if got.ContentEncoding != 0 || !bytes.Equal(got.UpdateBin, rows[0].UpdateBin) {
+ t.Fatal("failed migration changed first row")
+ }
+ var count int64
+ if err := app.db.G.Model(&contentSealing{}).Count(&count).Error; err != nil {
+ t.Fatal(err)
+ }
+ if count != 0 {
+ t.Fatal("failed migration activated sealing")
+ }
+}
+
+func TestContentRowsRefusePlaintextDowngrade(t *testing.T) {
+ c, err := newContentCipher("", "development")
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, encoding := range []int{0, 2, -1} {
+ if err := c.openUpdate("", &DocUpdate{ContentEncoding: encoding, UpdateBin: []byte("plaintext")}); !errors.Is(err, errContentEncoding) {
+ t.Fatalf("update encoding %d: %v", encoding, err)
+ }
+ if err := c.openSnapshot("", &DocSnapshot{ContentEncoding: encoding, State: []byte("plaintext")}); !errors.Is(err, errContentEncoding) {
+ t.Fatalf("snapshot encoding %d: %v", encoding, err)
+ }
+ }
+}
+
+func TestContentScrubRemovesLegacyBytesFromDatabaseAndWAL(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "docs.db")
+ app, err := NewForTestWith(t, Config{DBPath: path})
+ if err != nil {
+ t.Fatal(err)
+ }
+ doc, err := app.Store().Create(context.Background(), app.TestPerson(t, "owner@example.com"), "Scrub")
+ if err != nil {
+ t.Fatal(err)
+ }
+ canary := []byte("CONTENT-SCRUB-CANARY-55de8635-70b3-488a")
+ row := DocUpdate{DocID: doc.ID, Seq: 1, UpdateBin: canary, CreatedAt: time.Now().UTC()}
+ if err := app.db.G.Create(&row).Error; err != nil {
+ t.Fatal(err)
+ }
+ wal, err := os.ReadFile(path + "-wal")
+ if err != nil || !bytes.Contains(wal, canary) {
+ t.Fatalf("legacy WAL canary absent: %v", err)
+ }
+ c, err := newContentCipher("", "development")
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx := context.Background()
+ if err := convertContent(ctx, app.db.G, c); err != nil {
+ t.Fatal(err)
+ }
+ if err := scrubContent(ctx, app.db, c); err != nil {
+ t.Fatal(err)
+ }
+ for _, filename := range []string{path, path + "-wal"} {
+ body, err := os.ReadFile(filename)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if bytes.Contains(body, canary) {
+ t.Fatalf("plaintext retained in %s", filename)
+ }
+ }
+ state, err := readContentSealing(ctx, app.db.G, c)
+ if err != nil || state == nil || !state.Scrubbed {
+ t.Fatalf("scrub state: %#v, %v", state, err)
+ }
+}
diff --git a/internal/docs/content_rows.go b/internal/docs/content_rows.go
new file mode 100644
index 0000000..9076e32
--- /dev/null
+++ b/internal/docs/content_rows.go
@@ -0,0 +1,47 @@
+package docs
+
+import "fmt"
+
+func (c *contentCipher) sealUpdate(guid string, row *DocUpdate) error {
+ sealed, err := c.seal(contentUpdateKind, guid, row.Seq, 0, row.UpdateBin)
+ if err != nil {
+ return fmt.Errorf("docs: seal update %d: %w", row.ID, err)
+ }
+ row.UpdateBin = sealed
+ row.ContentEncoding = contentEncoding
+ return nil
+}
+
+func (c *contentCipher) openUpdate(guid string, row *DocUpdate) error {
+ if row.ContentEncoding != contentEncoding {
+ return errContentEncoding
+ }
+ plain, err := c.open(contentUpdateKind, guid, row.Seq, 0, row.UpdateBin)
+ if err != nil {
+ return fmt.Errorf("docs: open update %d: %w", row.ID, err)
+ }
+ row.UpdateBin = plain
+ return nil
+}
+
+func (c *contentCipher) sealSnapshot(guid string, row *DocSnapshot) error {
+ sealed, err := c.seal(contentSnapshotKind, guid, row.Seq, row.ABIVersion, row.State)
+ if err != nil {
+ return fmt.Errorf("docs: seal snapshot %d: %w", row.ID, err)
+ }
+ row.State = sealed
+ row.ContentEncoding = contentEncoding
+ return nil
+}
+
+func (c *contentCipher) openSnapshot(guid string, row *DocSnapshot) error {
+ if row.ContentEncoding != contentEncoding {
+ return errContentEncoding
+ }
+ plain, err := c.open(contentSnapshotKind, guid, row.Seq, row.ABIVersion, row.State)
+ if err != nil {
+ return fmt.Errorf("docs: open snapshot %d: %w", row.ID, err)
+ }
+ row.State = plain
+ return nil
+}
diff --git a/internal/docs/content_scrub.go b/internal/docs/content_scrub.go
new file mode 100644
index 0000000..be113f7
--- /dev/null
+++ b/internal/docs/content_scrub.go
@@ -0,0 +1,52 @@
+package docs
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+
+ "github.com/carlosframework/rastrillo/db"
+)
+
+func scrubContent(ctx context.Context, d *db.DB, c *contentCipher) error {
+ state, err := readContentSealing(ctx, d.G, c)
+ if err != nil {
+ return err
+ }
+ if state == nil {
+ return errContentState
+ }
+ if err := convertContent(ctx, d.G, c); err != nil {
+ return err
+ }
+ conn, err := d.Writer().Conn(ctx)
+ if err != nil {
+ return err
+ }
+ defer conn.Close()
+ if _, err := conn.ExecContext(ctx, "UPDATE content_sealing SET scrubbed = 0 WHERE id = 1"); err != nil {
+ return err
+ }
+ if err := truncateContentWAL(ctx, conn); err != nil {
+ return err
+ }
+ if _, err := conn.ExecContext(ctx, "VACUUM"); err != nil {
+ return err
+ }
+ if err := truncateContentWAL(ctx, conn); err != nil {
+ return err
+ }
+ _, err = conn.ExecContext(ctx, "UPDATE content_sealing SET scrubbed = 1 WHERE id = 1")
+ return err
+}
+
+func truncateContentWAL(ctx context.Context, conn *sql.Conn) error {
+ var busy, frames, checkpointed int
+ if err := conn.QueryRowContext(ctx, "PRAGMA wal_checkpoint(TRUNCATE)").Scan(&busy, &frames, &checkpointed); err != nil {
+ return err
+ }
+ if busy != 0 || frames != checkpointed {
+ return fmt.Errorf("docs: checkpoint incomplete (%d, %d, %d)", busy, frames, checkpointed)
+ }
+ return nil
+}
diff --git a/internal/docs/migrations/0007_content_encoding.sql b/internal/docs/migrations/0007_content_encoding.sql
new file mode 100644
index 0000000..8b90a14
--- /dev/null
+++ b/internal/docs/migrations/0007_content_encoding.sql
@@ -0,0 +1,10 @@
+ALTER TABLE doc_updates ADD COLUMN content_encoding INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE doc_snapshots ADD COLUMN content_encoding INTEGER NOT NULL DEFAULT 0;
+
+CREATE TABLE IF NOT EXISTS content_sealing (
+ id INTEGER PRIMARY KEY CHECK (id = 1),
+ encoding INTEGER NOT NULL,
+ key_fingerprint TEXT NOT NULL,
+ key_check BLOB NOT NULL,
+ scrubbed INTEGER NOT NULL DEFAULT 0
+);
diff --git a/internal/docs/models.go b/internal/docs/models.go
index c95c8d6..17301af 100644
--- a/internal/docs/models.go
+++ b/internal/docs/models.go
@@ -89,7 +89,8 @@ type DocUpdate struct {
DocID int64 `gorm:"column:doc_id;not null;index"`
Seq int64 `gorm:"column:seq;not null"`
- UpdateBin []byte `gorm:"column:update_bin;not null"`
+ UpdateBin []byte `gorm:"column:update_bin;not null"`
+ ContentEncoding int `gorm:"column:content_encoding;not null"`
CreatedAt time.Time `gorm:"column:created_at;not null"`
}
@@ -111,8 +112,9 @@ type DocSnapshot struct {
DocID int64 `gorm:"column:doc_id;not null;index"`
Seq int64 `gorm:"column:seq;not null"`
- State []byte `gorm:"column:state;not null"`
- ABIVersion uint32 `gorm:"column:abi_version;not null"`
+ State []byte `gorm:"column:state;not null"`
+ ContentEncoding int `gorm:"column:content_encoding;not null"`
+ ABIVersion uint32 `gorm:"column:abi_version;not null"`
CreatedAt time.Time `gorm:"column:created_at;not null"`
}