| 1 | package dbdoor |
| 2 | |
| 3 | import ( |
| 4 | "context" |
| 5 | "errors" |
| 6 | "io" |
| 7 | "os" |
| 8 | "path/filepath" |
| 9 | "sync" |
| 10 | "testing" |
| 11 | "time" |
| 12 | |
| 13 | "github.com/benbjohnson/litestream" |
| 14 | lsfile "github.com/benbjohnson/litestream/file" |
| 15 | "github.com/superfly/ltx" |
| 16 | |
| 17 | "github.com/carlosframework/platform/internal/dbdoor/dbdoortest" |
| 18 | ) |
| 19 | |
| 20 | // metaSpy records every useMetadata the listing passes down: the box must |
| 21 | // list LastModified as CreatedAt, exactly as the console's door does |
| 22 | // (listReplicaLevel), never S3 metadata timestamps (spec §6, T6). |
| 23 | type metaSpy struct { |
| 24 | ReplicaReader |
| 25 | sawMetadata bool |
| 26 | } |
| 27 | |
| 28 | func (m *metaSpy) LTXFiles(ctx context.Context, level int, seek ltx.TXID, useMetadata bool) (ltx.FileIterator, error) { |
| 29 | if useMetadata { |
| 30 | m.sawMetadata = true |
| 31 | } |
| 32 | return m.ReplicaReader.LTXFiles(ctx, level, seek, useMetadata) |
| 33 | } |
| 34 | |
| 35 | func TestListLevelIsTheDoorsListing(t *testing.T) { |
| 36 | t.Parallel() |
| 37 | dir := t.TempDir() |
| 38 | dbdoortest.Replicate(t, dir, 3) |
| 39 | spy := &metaSpy{ReplicaReader: lsfile.NewReplicaClient(dir)} |
| 40 | got, err := ListLevel(context.Background(), spy, 0) |
| 41 | if err != nil { |
| 42 | t.Fatal(err) |
| 43 | } |
| 44 | want := collect(t, NewClient(newTestDoor(t, dir).do), 0, 0) |
| 45 | if len(got.Files) != len(want) || spy.sawMetadata { |
| 46 | t.Fatalf("ListLevel = %d files (metadata %v), the door lists %d", len(got.Files), spy.sawMetadata, len(want)) |
| 47 | } |
| 48 | for i := range want { |
| 49 | f := got.Files[i] |
| 50 | if f.MinTXID != uint64(want[i].MinTXID) || f.MaxTXID != uint64(want[i].MaxTXID) || f.Size != want[i].Size || !f.CreatedAt.Equal(want[i].CreatedAt) { |
| 51 | t.Fatalf("file %d: %+v, the door lists %+v", i, f, want[i]) |
| 52 | } |
| 53 | } |
| 54 | empty, err := ListLevel(context.Background(), spy, 5) |
| 55 | if err != nil || empty.Files == nil || len(empty.Files) != 0 { |
| 56 | t.Fatalf("an empty level = %+v, %v; want an empty list", empty, err) |
| 57 | } |
| 58 | } |
| 59 | |
| 60 | // PlanFrom plans what RestoreAsOf through the door restores: the same |
| 61 | // transaction, the same time, the same size, for the newest copy and as of |
| 62 | // a time. |
| 63 | func TestPlanFromMatchesTheDoorsRestore(t *testing.T) { |
| 64 | dir := t.TempDir() |
| 65 | src := dbdoortest.Replicate(t, dir, 2) |
| 66 | at := pause() |
| 67 | addBatch(t, src, dir, 2) |
| 68 | for _, when := range []time.Time{{}, at} { |
| 69 | plan, err := PlanFrom(context.Background(), lsfile.NewReplicaClient(dir), when) |
| 70 | if err != nil { |
| 71 | t.Fatal(err) |
| 72 | } |
| 73 | got, err := RestoreAsOf(context.Background(), NewClient(newTestDoor(t, dir).do), t.TempDir(), when) |
| 74 | if err != nil { |
| 75 | t.Fatal(err) |
| 76 | } |
| 77 | st, err := os.Stat(got.Path) |
| 78 | if err != nil { |
| 79 | t.Fatal(err) |
| 80 | } |
| 81 | os.RemoveAll(got.Dir) |
| 82 | if plan.TXID != got.TXID || !plan.At.Equal(got.At) || plan.RestoredBytes != st.Size() || len(plan.Planned) != plan.Files || plan.Files == 0 { |
| 83 | t.Fatalf("as of %v: plan %+v; the door restored txid %d at %v, %d bytes", when, plan, got.TXID, got.At, st.Size()) |
| 84 | } |
| 85 | } |
| 86 | } |
| 87 | |
| 88 | func TestPlanFromBeforeHistoryNamesTheOldestPoint(t *testing.T) { |
| 89 | t.Parallel() |
| 90 | before := time.Now().Add(-time.Hour) |
| 91 | dir := t.TempDir() |
| 92 | dbdoortest.Replicate(t, dir, 2) |
| 93 | _, err := PlanFrom(context.Background(), lsfile.NewReplicaClient(dir), before) |
| 94 | var bh *BeforeHistoryError |
| 95 | if !errors.As(err, &bh) { |
| 96 | t.Fatalf("PlanFrom before history = %v", err) |
| 97 | } |
| 98 | _, derr := RestoreAsOf(context.Background(), NewClient(newTestDoor(t, dir).do), t.TempDir(), before) |
| 99 | var dbh *BeforeHistoryError |
| 100 | if !errors.As(derr, &dbh) || !dbh.Oldest.Equal(bh.Oldest) { |
| 101 | t.Fatalf("the door says %v, the plan says %v", derr, bh.Oldest) |
| 102 | } |
| 103 | } |
| 104 | |
| 105 | func TestPlanFromAnEmptyReplica(t *testing.T) { |
| 106 | t.Parallel() |
| 107 | _, err := PlanFrom(context.Background(), lsfile.NewReplicaClient(t.TempDir()), time.Time{}) |
| 108 | if !errors.Is(err, ErrNothingReplicated) { |
| 109 | t.Fatalf("PlanFrom on nothing = %v", err) |
| 110 | } |
| 111 | } |
| 112 | |
| 113 | // RestoreLocal restores exactly the planned files, copied into a fresh file |
| 114 | // replica the way the worker writes them, to the plan's transaction. |
| 115 | func TestRestoreLocalRestoresThePlan(t *testing.T) { |
| 116 | t.Parallel() |
| 117 | dir := t.TempDir() |
| 118 | dbdoortest.Replicate(t, dir, 4) |
| 119 | src := lsfile.NewReplicaClient(dir) |
| 120 | plan, err := PlanFrom(context.Background(), src, time.Time{}) |
| 121 | if err != nil { |
| 122 | t.Fatal(err) |
| 123 | } |
| 124 | root := t.TempDir() |
| 125 | for _, f := range plan.Planned { |
| 126 | rc, err := src.OpenLTXFile(context.Background(), f.Level, ltx.TXID(f.MinTXID), ltx.TXID(f.MaxTXID), 0, 0) |
| 127 | if err != nil { |
| 128 | t.Fatal(err) |
| 129 | } |
| 130 | path := litestream.LTXFilePath(root, f.Level, ltx.TXID(f.MinTXID), ltx.TXID(f.MaxTXID)) |
| 131 | if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil { |
| 132 | t.Fatal(err) |
| 133 | } |
| 134 | out, err := os.Create(path) |
| 135 | if err != nil { |
| 136 | t.Fatal(err) |
| 137 | } |
| 138 | if _, err := io.Copy(out, rc); err != nil { |
| 139 | t.Fatal(err) |
| 140 | } |
| 141 | rc.Close() |
| 142 | out.Close() |
| 143 | } |
| 144 | db := filepath.Join(t.TempDir(), "copy.db") |
| 145 | if err := RestoreLocal(context.Background(), lsfile.NewReplicaClient(root), db, plan.TXID); err != nil { |
| 146 | t.Fatal(err) |
| 147 | } |
| 148 | if rows := countRows(t, db); rows != 4*dbdoortest.Rows { |
| 149 | t.Fatalf("restored %d rows", rows) |
| 150 | } |
| 151 | st, _ := os.Stat(db) |
| 152 | if st.Size() != plan.RestoredBytes { |
| 153 | t.Fatalf("restored %d bytes, planned %d", st.Size(), plan.RestoredBytes) |
| 154 | } |
| 155 | } |
| 156 | |
| 157 | // countingReads is a litestream.ReplicaClient over a file replica that |
| 158 | // cancels a context on the first byte read and counts every read made after |
| 159 | // it, so a test can see whether a restore keeps reading once cancelled. |
| 160 | type countingReads struct { |
| 161 | litestream.ReplicaClient |
| 162 | cancel context.CancelFunc |
| 163 | mu sync.Mutex |
| 164 | reads int |
| 165 | after int |
| 166 | } |
| 167 | |
| 168 | func (c *countingReads) OpenLTXFile(ctx context.Context, level int, lo, hi ltx.TXID, off, size int64) (io.ReadCloser, error) { |
| 169 | rc, err := c.ReplicaClient.OpenLTXFile(ctx, level, lo, hi, off, size) |
| 170 | if err != nil { |
| 171 | return nil, err |
| 172 | } |
| 173 | return &countedReader{ReadCloser: rc, c: c}, nil |
| 174 | } |
| 175 | |
| 176 | type countedReader struct { |
| 177 | io.ReadCloser |
| 178 | c *countingReads |
| 179 | } |
| 180 | |
| 181 | func (r *countedReader) Read(p []byte) (int, error) { |
| 182 | r.c.mu.Lock() |
| 183 | r.c.reads++ |
| 184 | if r.c.reads == 1 { |
| 185 | r.c.cancel() |
| 186 | } else { |
| 187 | r.c.after++ |
| 188 | } |
| 189 | r.c.mu.Unlock() |
| 190 | return r.ReadCloser.Read(p) |
| 191 | } |
| 192 | |
| 193 | // Astra round 1 on plan 3, finding 2: litestream's local compaction checks |
| 194 | // no context, so RestoreLocal's replica reads must: once the restore is |
| 195 | // cancelled (Ctrl-C, or the read's deadline), no further byte is read and |
| 196 | // nothing is left at out. Mutation: RestoreLocal reads the client without |
| 197 | // its context wrapper (red: reads continue after the cancel). |
| 198 | func TestRestoreLocalStopsReadingOnceCancelled(t *testing.T) { |
| 199 | t.Parallel() |
| 200 | dir := t.TempDir() |
| 201 | dbdoortest.Replicate(t, dir, 4) |
| 202 | ctx, cancel := context.WithCancel(context.Background()) |
| 203 | defer cancel() |
| 204 | c := &countingReads{ReplicaClient: lsfile.NewReplicaClient(dir), cancel: cancel} |
| 205 | plan, err := PlanFrom(context.Background(), lsfile.NewReplicaClient(dir), time.Time{}) |
| 206 | if err != nil { |
| 207 | t.Fatal(err) |
| 208 | } |
| 209 | db := filepath.Join(t.TempDir(), "copy.db") |
| 210 | err = RestoreLocal(ctx, c, db, plan.TXID) |
| 211 | if !errors.Is(err, context.Canceled) { |
| 212 | t.Fatalf("a cancelled restore answered %v", err) |
| 213 | } |
| 214 | if c.after != 0 { |
| 215 | t.Fatalf("%d replica reads after the cancel", c.after) |
| 216 | } |
| 217 | if _, err := os.Stat(db); !os.IsNotExist(err) { |
| 218 | t.Fatalf("something is at out: %v", err) |
| 219 | } |
| 220 | } |
| 221 | |
amadan