amadan

oficina / docs Public

Exercise process-crash recovery and blocked WAL scrub

Paul Campbell pushed by paul@keymail.dev 4bd6cca2256760d35264e134903e9b2d0207a8ec parent aab3abc
1 file changed, +133 −0
  • internal/docs/content_recovery_test.go +133 −0
diff --git a/internal/docs/content_recovery_test.go b/internal/docs/content_recovery_test.go
new file mode 100644
index 0000000..b87e3f3
--- /dev/null
+++ b/internal/docs/content_recovery_test.go
@@ -0,0 +1,133 @@
+package docs
+
+import (
+ "bytes"
+ "context"
+ "database/sql"
+ "errors"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "strings"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/rastrillo/db"
+ "gorm.io/gorm"
+)
+
+func TestContentScrubRefusesPinnedWALReader(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "docs.db")
+ app, err := NewForTestWith(t, Config{DBPath: path})
+ if err != nil {
+ t.Fatal(err)
+ }
+ c, err := newContentCipher("", "development")
+ if err != nil {
+ t.Fatal(err)
+ }
+ ctx := context.Background()
+ if err := convertContent(ctx, app.db.G, c); err != nil {
+ t.Fatal(err)
+ }
+ if err := scrubContent(ctx, app.db, c); err != nil {
+ t.Fatal(err)
+ }
+ reader, err := sql.Open("sqlite", path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer reader.Close()
+ tx, err := reader.BeginTx(ctx, &sql.TxOptions{ReadOnly: true})
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer tx.Rollback()
+ var scrubbed int
+ if err := tx.QueryRow("SELECT scrubbed FROM content_sealing WHERE id=1").Scan(&scrubbed); err != nil {
+ t.Fatal(err)
+ }
+ if scrubbed != 1 {
+ t.Fatal("reader did not pin scrubbed state")
+ }
+ if _, err := app.db.Writer().Exec("PRAGMA busy_timeout=1"); err != nil {
+ t.Fatal(err)
+ }
+ err = scrubContent(ctx, app.db, c)
+ if err == nil || !strings.Contains(err.Error(), "checkpoint incomplete") {
+ t.Fatalf("pinned reader: %v", err)
+ }
+ state, err := readContentSealing(ctx, app.db.G, c)
+ if err != nil || state == nil || state.Scrubbed {
+ t.Fatalf("failed scrub marker: %#v, %v", state, err)
+ }
+}
+
+func TestContentMigrationRecoversAfterProcessExit(t *testing.T) {
+ ctx := context.Background()
+ c, err := newContentCipher("", "development")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if path := os.Getenv("DOCS_SEAL_CRASH_FIXTURE"); path != "" {
+ d, err := db.Open(path, nil)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer d.Close()
+ if err := d.G.Callback().Update().After("gorm:commit_or_rollback_transaction").Register("docs:test-crash", func(tx *gorm.DB) {
+ if tx.Statement.Table == "doc_updates" && tx.Error == nil && tx.RowsAffected == 1 {
+ os.Exit(73)
+ }
+ }); err != nil {
+ t.Fatal(err)
+ }
+ if err := convertContent(ctx, d.G, c); err != nil {
+ t.Fatal(err)
+ }
+ t.Fatal("conversion never reached crash point")
+ }
+ path := filepath.Join(t.TempDir(), "docs.db")
+ app, err := NewForTestWith(t, Config{DBPath: path})
+ if err != nil {
+ t.Fatal(err)
+ }
+ doc, err := app.Store().Create(ctx, app.TestPerson(t, "owner@example.com"), "Crash recovery")
+ if err != nil {
+ t.Fatal(err)
+ }
+ row := DocUpdate{DocID: doc.ID, Seq: 1, UpdateBin: validUpdate(t), CreatedAt: time.Now().UTC()}
+ if err := app.db.G.Create(&row).Error; err != nil {
+ t.Fatal(err)
+ }
+ command := exec.Command(os.Args[0], "-test.run=^TestContentMigrationRecoversAfterProcessExit$", "-test.count=1")
+ command.Env = append(os.Environ(), "DOCS_SEAL_CRASH_FIXTURE="+path)
+ output, err := command.CombinedOutput()
+ var exited *exec.ExitError
+ if !errors.As(err, &exited) || exited.ExitCode() != 73 {
+ t.Fatalf("child did not exit at write: %v\n%s", err, output)
+ }
+ var recovered DocUpdate
+ if err := app.db.G.First(&recovered, row.ID).Error; err != nil {
+ t.Fatal(err)
+ }
+ if recovered.ContentEncoding != 0 || !bytes.Equal(recovered.UpdateBin, row.UpdateBin) {
+ t.Fatal("process exit committed a partial conversion")
+ }
+ state, err := readContentSealing(ctx, app.db.G, c)
+ if err != nil || state != nil {
+ t.Fatalf("crash activated sealing: %#v, %v", state, err)
+ }
+ if err := convertContent(ctx, app.db.G, c); err != nil {
+ t.Fatalf("retry: %v", err)
+ }
+ if err := app.db.G.First(&recovered, row.ID).Error; err != nil {
+ t.Fatal(err)
+ }
+ if err := c.openUpdate(doc.GUID, &recovered); err != nil {
+ t.Fatal(err)
+ }
+ if !bytes.Equal(recovered.UpdateBin, row.UpdateBin) {
+ t.Fatal("retry lost content")
+ }
+}