| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "database/sql" |
| 7 | "encoding/json" |
| 8 | "io" |
| 9 | "net/http" |
| 10 | "os" |
| 11 | "path/filepath" |
| 12 | "strings" |
| 13 | "testing" |
| 14 | "time" |
| 15 | |
| 16 | lsfile "github.com/benbjohnson/litestream/file" |
| 17 | "github.com/carlosframework/platform/internal/dbdoor" |
| 18 | "github.com/carlosframework/platform/internal/dbdoor/dbdoortest" |
| 19 | "github.com/carlosframework/platform/internal/genselect" |
| 20 | "github.com/carlosframework/platform/internal/hostclaim" |
| 21 | "github.com/superfly/ltx" |
| 22 | ) |
| 23 | |
| 24 | func TestDoorPlanUnnamedHomeRegion(t *testing.T) { |
| 25 | for _, unfinished := range []bool{false, true} { |
| 26 | t.Run(map[bool]string{false: "origin", true: "unfinished"}[unfinished], func(t *testing.T) { |
| 27 | fx := dbFixture(t) |
| 28 | model := enableDoorPlan(t, fx) |
| 29 | fx.srv.cfg.HomeRegion = "" |
| 30 | fx.srv.cfg.HibernateRegion = "eu-west-1" |
| 31 | if fx.srv.cfg.Region != "" { |
| 32 | t.Fatal("fixture has a placement") |
| 33 | } |
| 34 | if unfinished { |
| 35 | obs, err := hostclaim.Read(t.Context(), fx.rel, dbTestHost) |
| 36 | if err != nil { |
| 37 | t.Fatal(err) |
| 38 | } |
| 39 | c := cloneDoorClaim(obs.Claim) |
| 40 | c.Holder.Generation = "next" |
| 41 | c.Holder.Admissions = []hostclaim.AdmissionRecord{{Generation: "next", Predecessor: hostclaim.RootGeneration, Region: "eu-west-1", PredecessorRegion: "eu-west-1", Mode: "cold"}} |
| 42 | writeDoorClaim(t, fx.srv, dbRequest{host: dbTestHost}, c) |
| 43 | } |
| 44 | model.Put("data-test", dbVersionKey("instances/"+dbTestHost, 0, 1, 1), doorLTXHeader(t, 1, 1)) |
| 45 | status, body := getRaw(t, fx.url(dbdoor.ListPath("bab", "titogo", dbTestHost, 0)), fx.member) |
| 46 | if status != http.StatusOK { |
| 47 | t.Fatalf("unnamed home listing: %d %s", status, body) |
| 48 | } |
| 49 | }) |
| 50 | } |
| 51 | } |
| 52 | |
| 53 | func TestDoorPlanRegionStartup(t *testing.T) { |
| 54 | fx := dbFixture(t) |
| 55 | for _, mode := range []string{"unresolved", "bucket-region", "named-home", "placement", "flag-off", "door-off"} { |
| 56 | t.Run(mode, func(t *testing.T) { |
| 57 | cfg := fx.srv.cfg |
| 58 | cfg.HostClaim = true |
| 59 | cfg.Region, cfg.HomeRegion, cfg.HibernateRegion = "", "", "" |
| 60 | switch mode { |
| 61 | case "bucket-region": |
| 62 | cfg.HibernateRegion = "eu-west-1" |
| 63 | case "named-home": |
| 64 | cfg.HomeRegion = "ie" |
| 65 | case "placement": |
| 66 | cfg.Region = "uk" |
| 67 | case "flag-off": |
| 68 | cfg.HostClaim = false |
| 69 | case "door-off": |
| 70 | cfg.HibernateBucket = "" |
| 71 | } |
| 72 | st := fx.st |
| 73 | if cfg.HostClaim { |
| 74 | var err error |
| 75 | st, err = OpenGated(filepath.Join(t.TempDir(), "gated.db")) |
| 76 | if err != nil { |
| 77 | t.Fatal(err) |
| 78 | } |
| 79 | t.Cleanup(func() { st.Close() }) |
| 80 | } |
| 81 | _, err := NewServer(cfg, st) |
| 82 | if mode == "unresolved" { |
| 83 | if err == nil || !strings.Contains(err.Error(), "region") { |
| 84 | t.Fatalf("startup without a region: %v", err) |
| 85 | } |
| 86 | } else if err != nil { |
| 87 | t.Fatal(err) |
| 88 | } |
| 89 | }) |
| 90 | } |
| 91 | } |
| 92 | |
| 93 | func TestDoorPlanPinnedSnapshotCRC(t *testing.T) { |
| 94 | for _, mode := range []string{"match", "mismatch", "zero", "historical", "successor"} { |
| 95 | t.Run(mode, func(t *testing.T) { |
| 96 | fx := dbFixture(t) |
| 97 | fx.replicate(t, dbTestHost, 2) |
| 98 | replica := lsfile.NewReplicaClient(filepath.Join(fx.root, "instances", dbTestHost)) |
| 99 | var readers []io.Reader |
| 100 | for _, f := range listFiles(t, replica, 0) { |
| 101 | b, err := os.ReadFile(replica.LTXFilePath(0, f.MinTXID, f.MaxTXID)) |
| 102 | if err != nil { |
| 103 | t.Fatal(err) |
| 104 | } |
| 105 | readers = append(readers, bytes.NewReader(b)) |
| 106 | } |
| 107 | var snapshot bytes.Buffer |
| 108 | compactor, err := ltx.NewCompactor(&snapshot, readers) |
| 109 | if err != nil { |
| 110 | t.Fatal(err) |
| 111 | } |
| 112 | compactor.HeaderFlags = ltx.HeaderFlagNoChecksum |
| 113 | if err := compactor.Compact(t.Context()); err != nil { |
| 114 | t.Fatal(err) |
| 115 | } |
| 116 | dec := ltx.NewDecoder(bytes.NewReader(snapshot.Bytes())) |
| 117 | if err := dec.Verify(); err != nil { |
| 118 | t.Fatal(err) |
| 119 | } |
| 120 | crc := uint64(dec.Trailer().FileChecksum) |
| 121 | maxTXID := uint64(dec.Header().MaxTXID) |
| 122 | model := enableDoorPlan(t, fx) |
| 123 | obs, err := hostclaim.Read(t.Context(), fx.rel, dbTestHost) |
| 124 | if err != nil { |
| 125 | t.Fatal(err) |
| 126 | } |
| 127 | c := cloneDoorClaim(obs.Claim) |
| 128 | c.Holder.Generation = "live" |
| 129 | loc := c.Holder.Location |
| 130 | key := dbVersionKey(loc+"/g-live", 9, 1, maxTXID) |
| 131 | id := model.Put("data-test", key, snapshot.Bytes()) |
| 132 | pin := genselect.Pin{SnapshotKey: key, SnapshotVersion: id, SnapshotTXID: maxTXID, SnapshotCRC64: crc} |
| 133 | switch mode { |
| 134 | case "mismatch": |
| 135 | pin.SnapshotCRC64 ^= 1 |
| 136 | case "zero": |
| 137 | pin.SnapshotCRC64 = 0 |
| 138 | case "historical": |
| 139 | model.Put("data-test", key, []byte("late replacement")) |
| 140 | case "successor": |
| 141 | pin.SnapshotKey = dbVersionKey(loc+"/g-live", 9, 1, 1) |
| 142 | pin.SnapshotTXID = 1 |
| 143 | pin.SnapshotVersion = "retained-marker-only" |
| 144 | pin.SnapshotCRC64 ^= 1 |
| 145 | } |
| 146 | marker := genselect.Marker{Op: "live", Host: dbTestHost, Incarnation: c.Holder.Incarnation, Box: "box", InstalledAt: time.Unix(1, 0).UTC(), Pin: pin} |
| 147 | raw, err := json.Marshal(marker) |
| 148 | if err != nil { |
| 149 | t.Fatal(err) |
| 150 | } |
| 151 | model.Put("data-test", loc+"/generation.json", raw) |
| 152 | writeDoorClaim(t, fx.srv, dbRequest{host: dbTestHost}, c) |
| 153 | do := func(ctx context.Context, tail string, h http.Header) (*http.Response, error) { |
| 154 | req, err := http.NewRequestWithContext(ctx, "GET", fx.url(dbdoor.Base("bab", "titogo", dbTestHost)+tail), nil) |
| 155 | if err != nil { |
| 156 | return nil, err |
| 157 | } |
| 158 | req.Header = h.Clone() |
| 159 | req.Header.Set("Authorization", "Bearer "+fx.member) |
| 160 | return http.DefaultClient.Do(req) |
| 161 | } |
| 162 | parent := t.TempDir() |
| 163 | got, err := dbdoor.RestoreTo(t.Context(), dbdoor.NewClient(do), parent) |
| 164 | if mode == "mismatch" || mode == "zero" { |
| 165 | if got != nil { |
| 166 | os.RemoveAll(got.Dir) |
| 167 | t.Fatal("restored a valid snapshot whose CRC differs from the marker") |
| 168 | } |
| 169 | if err == nil || !strings.Contains(err.Error(), dbdoor.ErrCheckFailed.Error()) { |
| 170 | t.Fatalf("want member-safe check refusal: %v", err) |
| 171 | } |
| 172 | entries, readErr := os.ReadDir(parent) |
| 173 | if readErr != nil || len(entries) != 0 { |
| 174 | t.Fatalf("failed restore left files: %v %v", entries, readErr) |
| 175 | } |
| 176 | } else { |
| 177 | if err != nil { |
| 178 | t.Fatal(err) |
| 179 | } |
| 180 | defer os.RemoveAll(got.Dir) |
| 181 | db, err := sql.Open("sqlite", "file:"+got.Path+"?mode=ro") |
| 182 | if err != nil { |
| 183 | t.Fatal(err) |
| 184 | } |
| 185 | defer db.Close() |
| 186 | var rows int |
| 187 | if err := db.QueryRow("SELECT count(*) FROM t").Scan(&rows); err != nil || rows != 2*dbdoortest.Rows { |
| 188 | t.Fatalf("rows %d: %v", rows, err) |
| 189 | } |
| 190 | } |
| 191 | res, err := do(t.Context(), dbdoor.ListTail(9), http.Header{}) |
| 192 | if err != nil { |
| 193 | t.Fatal(err) |
| 194 | } |
| 195 | defer res.Body.Close() |
| 196 | var list struct { |
| 197 | Files []struct { |
| 198 | Handle string |
| 199 | SnapshotCRC64 *uint64 `json:"snapshot_crc64"` |
| 200 | } |
| 201 | } |
| 202 | if err := json.NewDecoder(res.Body).Decode(&list); err != nil || len(list.Files) != 1 { |
| 203 | t.Fatalf("listing: %+v %v", list, err) |
| 204 | } |
| 205 | h, err := verifyDBHandle(fx.srv.dbHandleKey, list.Files[0].Handle, time.Now()) |
| 206 | if err != nil { |
| 207 | t.Fatal(err) |
| 208 | } |
| 209 | raw, err = json.Marshal(h) |
| 210 | if err != nil { |
| 211 | t.Fatal(err) |
| 212 | } |
| 213 | var signed struct { |
| 214 | SnapshotCRC64 *uint64 `json:"snapshot_crc64"` |
| 215 | } |
| 216 | if err := json.Unmarshal(raw, &signed); err != nil { |
| 217 | t.Fatal(err) |
| 218 | } |
| 219 | for _, value := range []*uint64{list.Files[0].SnapshotCRC64, signed.SnapshotCRC64} { |
| 220 | if mode == "successor" { |
| 221 | if value != nil { |
| 222 | t.Fatal("successor inherited old pin CRC") |
| 223 | } |
| 224 | } else if value == nil || *value != pin.SnapshotCRC64 { |
| 225 | t.Fatalf("lost expected CRC: %v, want %d", value, pin.SnapshotCRC64) |
| 226 | } |
| 227 | } |
| 228 | }) |
| 229 | } |
| 230 | } |
| 231 | |
amadan