| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "crypto/ecdsa" |
| 7 | "crypto/hmac" |
| 8 | "crypto/sha256" |
| 9 | "encoding/base64" |
| 10 | "encoding/json" |
| 11 | "errors" |
| 12 | "fmt" |
| 13 | "github.com/carlosframework/platform/internal/park" |
| 14 | "io" |
| 15 | "net/http" |
| 16 | "os" |
| 17 | "reflect" |
| 18 | "strconv" |
| 19 | "strings" |
| 20 | "time" |
| 21 | |
| 22 | "github.com/carlosframework/platform/internal/dbdoor" |
| 23 | "github.com/carlosframework/platform/internal/genselect" |
| 24 | "github.com/carlosframework/platform/internal/hostclaim" |
| 25 | "github.com/carlosframework/platform/internal/lease" |
| 26 | "github.com/carlosframework/platform/internal/provision" |
| 27 | "github.com/superfly/ltx" |
| 28 | ) |
| 29 | |
| 30 | const dbHandleTTL = 15 * time.Minute |
| 31 | |
| 32 | var ( |
| 33 | errDBNotAdopted = errors.New("host not adopted") |
| 34 | errDBClaimChanged = errors.New("host claim changed; retry the database pull") |
| 35 | errDBClaimClosed = errors.New("host is not accepting database reads") |
| 36 | errDBHandle = errors.New("invalid database file handle") |
| 37 | errDBHandleExpired = errors.New("database file handle expired; retry the database pull") |
| 38 | ) |
| 39 | |
| 40 | // dbFileHandle is encoded as this ordered JSON struct, never as a map. |
| 41 | // Account and App bind the capability to the authenticated route as well. |
| 42 | type dbFileHandle struct { |
| 43 | V int `json:"v"` |
| 44 | Host string `json:"host"` |
| 45 | Location string `json:"location"` |
| 46 | Key string `json:"key"` |
| 47 | VersionID string `json:"version_id"` |
| 48 | Incarnation string `json:"incarnation"` |
| 49 | Expiry int64 `json:"expiry"` |
| 50 | Account string `json:"account"` |
| 51 | App string `json:"app"` |
| 52 | Size int64 `json:"size"` |
| 53 | SnapshotCRC64 *uint64 `json:"snapshot_crc64,omitempty"` |
| 54 | } |
| 55 | |
| 56 | func dbHandleSecret(key *ecdsa.PrivateKey) []byte { |
| 57 | mac := hmac.New(sha256.New, key.D.FillBytes(make([]byte, 32))) |
| 58 | mac.Write([]byte("carlos-db-handle-key-v1\x00")) |
| 59 | return mac.Sum(nil) |
| 60 | } |
| 61 | |
| 62 | func dbHandleMAC(secret, body []byte) []byte { |
| 63 | mac := hmac.New(sha256.New, secret) |
| 64 | mac.Write([]byte("carlos-db-file-v1\x00")) |
| 65 | mac.Write(body) |
| 66 | return mac.Sum(nil) |
| 67 | } |
| 68 | |
| 69 | func signDBHandle(secret []byte, h dbFileHandle) (string, error) { |
| 70 | if len(secret) != sha256.Size { |
| 71 | return "", errDBHandle |
| 72 | } |
| 73 | body, err := json.Marshal(h) |
| 74 | if err != nil { |
| 75 | return "", err |
| 76 | } |
| 77 | return base64.RawURLEncoding.EncodeToString(body) + "." + base64.RawURLEncoding.EncodeToString(dbHandleMAC(secret, body)), nil |
| 78 | } |
| 79 | |
| 80 | func verifyDBHandle(secret []byte, token string, now time.Time) (dbFileHandle, error) { |
| 81 | var h dbFileHandle |
| 82 | if len(secret) != sha256.Size || len(token) > 8192 { |
| 83 | return h, errDBHandle |
| 84 | } |
| 85 | a, b, ok := strings.Cut(token, ".") |
| 86 | if !ok { |
| 87 | return h, errDBHandle |
| 88 | } |
| 89 | body, err := base64.RawURLEncoding.DecodeString(a) |
| 90 | if err != nil { |
| 91 | return h, errDBHandle |
| 92 | } |
| 93 | sig, err := base64.RawURLEncoding.DecodeString(b) |
| 94 | if err != nil || !hmac.Equal(sig, dbHandleMAC(secret, body)) { |
| 95 | return h, errDBHandle |
| 96 | } |
| 97 | dec := json.NewDecoder(bytes.NewReader(body)) |
| 98 | dec.DisallowUnknownFields() |
| 99 | if err := dec.Decode(&h); err != nil { |
| 100 | return dbFileHandle{}, errDBHandle |
| 101 | } |
| 102 | canonical, err := json.Marshal(h) |
| 103 | if err != nil || !bytes.Equal(body, canonical) || h.V != 1 || h.VersionID == "" || h.VersionID == "null" || h.Size < 0 { |
| 104 | return dbFileHandle{}, errDBHandle |
| 105 | } |
| 106 | if now.Unix() >= h.Expiry { |
| 107 | return dbFileHandle{}, errDBHandleExpired |
| 108 | } |
| 109 | return h, nil |
| 110 | } |
| 111 | |
| 112 | func dbLevelPrefix(location string, level int) string { |
| 113 | // litestream/s3 uses hex level directories, unlike its file replica. |
| 114 | return fmt.Sprintf("%s/%04x/", location, level) |
| 115 | } |
| 116 | |
| 117 | func dbVersionKey(location string, level int, lo, hi uint64) string { |
| 118 | return dbLevelPrefix(location, level) + ltx.FormatFilename(ltx.TXID(lo), ltx.TXID(hi)) |
| 119 | } |
| 120 | |
| 121 | func (s *Server) dbClaim(ctx context.Context, g dbRequest, rec provision.Record) (hostclaim.Observed, error) { |
| 122 | obs, err := hostclaim.Read(ctx, s.cfg.Releases, g.host) |
| 123 | if err != nil { |
| 124 | return hostclaim.Observed{}, err |
| 125 | } |
| 126 | if obs.Absent { |
| 127 | return obs, errDBNotAdopted |
| 128 | } |
| 129 | h := obs.Claim.Holder |
| 130 | if h == nil { |
| 131 | return obs, errDBClaimClosed |
| 132 | } |
| 133 | inc, err := hostclaim.Incarnation(g.acct.Sqid, g.app, g.host, rec.CreatedAt) |
| 134 | if err != nil { |
| 135 | return obs, err |
| 136 | } |
| 137 | if h.Family != "instance" || h.Account != g.acct.Sqid || h.App != g.app || h.Key != provision.RecordKey(g.acct.Sqid, g.app, g.host) || h.Incarnation != inc || !h.CreatedAt.Equal(rec.CreatedAt) { |
| 138 | return obs, os.ErrNotExist |
| 139 | } |
| 140 | switch obs.Claim.Op.Kind { |
| 141 | case hostclaim.KindDeleting, hostclaim.KindParking, hostclaim.KindPurging: |
| 142 | return obs, errDBClaimClosed |
| 143 | case hostclaim.KindMoving, hostclaim.KindLanding, hostclaim.KindReverting, hostclaim.KindRestoring, hostclaim.KindRelocating: |
| 144 | // Every move kind, in every phase, landing/committed and the |
| 145 | // opened phases included (moves spec §19.1): the door reopens at |
| 146 | // the close, when the claim is serving again. |
| 147 | return obs, errDBClaimClosed |
| 148 | } |
| 149 | if h.Location == "" { |
| 150 | return obs, errDBClaimClosed |
| 151 | } |
| 152 | return obs, nil |
| 153 | } |
| 154 | |
| 155 | func sameDBAdmission(a, b hostclaim.Observed) bool { |
| 156 | if a.Absent || b.Absent || a.Claim == nil || b.Claim == nil { |
| 157 | return false |
| 158 | } |
| 159 | // §6.5 permits unrelated chore revisions; compare all holder/op content, |
| 160 | // including stamp, session, admissions, journal and execution ownership. |
| 161 | return a.Host == b.Host && reflect.DeepEqual(a.Claim.Holder, b.Claim.Holder) && reflect.DeepEqual(a.Claim.Op, b.Claim.Op) |
| 162 | } |
| 163 | |
| 164 | func (s *Server) dbClaimFailure(w http.ResponseWriter, r *http.Request, g dbRequest, err error) { |
| 165 | switch { |
| 166 | case errors.Is(err, os.ErrNotExist): |
| 167 | http.NotFound(w, r) |
| 168 | case errors.Is(err, errDBNotAdopted), errors.Is(err, errDBClaimChanged), errors.Is(err, errDBClaimClosed): |
| 169 | w.Header().Set("Retry-After", "1") |
| 170 | jsonError(w, http.StatusConflict, err.Error()) |
| 171 | case errors.Is(err, errDBHandleExpired): |
| 172 | jsonError(w, http.StatusGone, err.Error()) |
| 173 | case errors.Is(err, errDBHandle): |
| 174 | jsonError(w, http.StatusBadRequest, err.Error()) |
| 175 | default: |
| 176 | s.dbStoreFailure(w, g, "claim/version read", err) |
| 177 | } |
| 178 | } |
| 179 | |
| 180 | func (cfg Config) claimPlacementRegion(code string) string { |
| 181 | region := park.PlacementRegion(code) |
| 182 | if region == park.HomeRegion && park.PlacementRegion(cfg.HomeRegion) == cfg.HomeRegion && cfg.HomeRegion != "" { |
| 183 | return cfg.HomeRegion |
| 184 | } |
| 185 | return region |
| 186 | } |
| 187 | |
| 188 | // dbClaimHomeRegion binds the old fallback only for this configured home |
| 189 | // bucket. Admission contents never choose the compatibility name. |
| 190 | func (cfg Config) dbClaimHomeRegion() string { |
| 191 | if cfg.HomeRegion != "" { |
| 192 | return cfg.HomeRegion |
| 193 | } |
| 194 | legacy := strings.TrimSpace(cfg.HibernateRegion) |
| 195 | if cfg.HostClaim && cfg.Region == "" && cfg.HibernateBucket != "" && legacy != "" && park.PlacementRegion(legacy) == legacy { |
| 196 | return legacy |
| 197 | } |
| 198 | return "" |
| 199 | } |
| 200 | |
| 201 | func (cfg Config) dbClaimRegion() string { |
| 202 | if cfg.HostClaim { |
| 203 | if cfg.Region == "" && cfg.HomeRegion == "" && strings.TrimSpace(cfg.HibernateRegion) == "" { |
| 204 | return "" |
| 205 | } |
| 206 | return cfg.claimPlacementRegion(cfg.Region) |
| 207 | } |
| 208 | for _, region := range []string{cfg.Region, cfg.HomeRegion, cfg.HibernateRegion} { |
| 209 | if region = strings.TrimSpace(region); region != "" { |
| 210 | return region |
| 211 | } |
| 212 | } |
| 213 | return "" |
| 214 | } |
| 215 | |
| 216 | func (s *Server) listDBVersions(ctx context.Context, g dbRequest, level int) (dbdoor.ListResponse, error) { |
| 217 | // Select the complete baseline once; the client retains it across levels. |
| 218 | out := dbdoor.ListResponse{Protocol: 2, Plan: true, Files: []dbdoor.LTXFile{}} |
| 219 | h := g.claim.Claim.Holder |
| 220 | plan, err := genselect.Select(ctx, s.cfg.DBVersions, genselect.Request{ |
| 221 | Bucket: s.cfg.HibernateBucket, Region: s.cfg.dbClaimRegion(), Host: g.host, Holder: park.HolderRegionView(*h, s.cfg.dbClaimHomeRegion(), s.cfg.dbClaimRegion()), |
| 222 | }) |
| 223 | if err != nil { |
| 224 | return dbdoor.ListResponse{}, errDBClaimChanged |
| 225 | } |
| 226 | for _, f := range plan.Files { |
| 227 | var crc *uint64 |
| 228 | if plan.Pin != nil && f.Key == plan.Pin.SnapshotKey && f.VersionID == plan.Pin.SnapshotVersion { |
| 229 | crc = &plan.Pin.SnapshotCRC64 |
| 230 | } |
| 231 | token, err := signDBHandle(s.dbHandleKey, dbFileHandle{V: 1, Host: g.host, Location: h.Location, Key: f.Key, VersionID: f.VersionID, Incarnation: h.Incarnation, Expiry: time.Now().Add(dbHandleTTL).Unix(), Account: g.acct.Sqid, App: g.app, Size: f.Size, SnapshotCRC64: crc}) |
| 232 | if err != nil { |
| 233 | return dbdoor.ListResponse{}, err |
| 234 | } |
| 235 | out.Files = append(out.Files, dbdoor.LTXFile{Level: f.Level, MinTXID: f.MinTXID, MaxTXID: f.MaxTXID, Size: f.Size, Handle: token, SnapshotCRC64: crc}) |
| 236 | } |
| 237 | o2, err := hostclaim.Read(ctx, s.cfg.Releases, g.host) |
| 238 | if err != nil { |
| 239 | return dbdoor.ListResponse{}, err |
| 240 | } |
| 241 | if !sameDBAdmission(g.claim, o2) { |
| 242 | return dbdoor.ListResponse{}, errDBClaimChanged |
| 243 | } |
| 244 | return out, nil |
| 245 | } |
| 246 | |
| 247 | func dbHandleCoordinate(h dbFileHandle, level int, lo, hi uint64) bool { |
| 248 | if h.Key == dbVersionKey(h.Location, level, lo, hi) { |
| 249 | return true |
| 250 | } |
| 251 | rel, ok := strings.CutPrefix(h.Key, h.Location+"/g-") |
| 252 | if !ok { |
| 253 | return false |
| 254 | } |
| 255 | gen, tail, ok := strings.Cut(rel, "/") |
| 256 | return ok && gen != "" && gen != "." && gen != ".." && gen != hostclaim.RootGeneration && !strings.ContainsAny(gen, "\\\x00") && tail == dbVersionKey("", level, lo, hi)[1:] |
| 257 | } |
| 258 | |
| 259 | func (s *Server) pinnedDBReader(r *http.Request, g dbRequest, level int, lo, hi uint64) (*dbVersionReader, error) { |
| 260 | h, err := verifyDBHandle(s.dbHandleKey, r.Header.Get(dbdoor.HandleHeader), time.Now()) |
| 261 | if err != nil { |
| 262 | return nil, err |
| 263 | } |
| 264 | holder := g.claim.Claim.Holder |
| 265 | if h.Host != g.host || h.Account != g.acct.Sqid || h.App != g.app || h.Location != holder.Location || h.Incarnation != holder.Incarnation || !dbHandleCoordinate(h, level, lo, hi) { |
| 266 | return nil, errDBHandle |
| 267 | } |
| 268 | return &dbVersionReader{store: s.cfg.DBVersions, bucket: s.cfg.HibernateBucket, handle: h}, nil |
| 269 | } |
| 270 | |
| 271 | // This adapter keeps the existing stream budget, watchdog and audit path. |
| 272 | // Its only object read names the signed version, including on every resume. |
| 273 | type dbVersionReader struct { |
| 274 | store lease.VersionedStore |
| 275 | bucket string |
| 276 | handle dbFileHandle |
| 277 | } |
| 278 | |
| 279 | func (v *dbVersionReader) LTXFiles(context.Context, int, ltx.TXID, bool) (ltx.FileIterator, error) { |
| 280 | return nil, errDBHandle |
| 281 | } |
| 282 | func (v *dbVersionReader) OpenLTXFile(ctx context.Context, level int, lo, hi ltx.TXID, offset, size int64) (io.ReadCloser, error) { |
| 283 | if !dbHandleCoordinate(v.handle, level, uint64(lo), uint64(hi)) { |
| 284 | return nil, errDBHandle |
| 285 | } |
| 286 | n, err := dbVersionRange(v.handle.Size, offset, size) |
| 287 | if err != nil { |
| 288 | return nil, err |
| 289 | } |
| 290 | var rc io.ReadCloser |
| 291 | ranged, ok := v.store.(lease.VersionRangeOpener) |
| 292 | if ok && n > 0 { |
| 293 | rc, err = ranged.OpenVersionRange(ctx, v.bucket, v.handle.Key, v.handle.VersionID, offset, n) |
| 294 | } else { |
| 295 | rc, err = v.store.OpenVersion(ctx, v.bucket, v.handle.Key, v.handle.VersionID) |
| 296 | } |
| 297 | if errors.Is(err, lease.ErrMirrorNotFound) { |
| 298 | return nil, os.ErrNotExist |
| 299 | } |
| 300 | if err != nil { |
| 301 | return nil, err |
| 302 | } |
| 303 | if !ok { |
| 304 | if _, err = io.CopyN(io.Discard, rc, offset); err != nil { |
| 305 | rc.Close() |
| 306 | return nil, err |
| 307 | } |
| 308 | } |
| 309 | return &dbRangeReader{Reader: io.LimitReader(rc, n), Closer: rc}, nil |
| 310 | } |
| 311 | |
| 312 | type dbRangeReader struct { |
| 313 | io.Reader |
| 314 | io.Closer |
| 315 | } |
| 316 | |
| 317 | func dbVersionRange(total, offset, size int64) (int64, error) { |
| 318 | if total < 0 || offset < 0 || size < 0 || offset > total || (offset == total && total != 0) { |
| 319 | return 0, errDBHandle |
| 320 | } |
| 321 | n := total - offset |
| 322 | if size != 0 && size < n { |
| 323 | n = size |
| 324 | } |
| 325 | return n, nil |
| 326 | } |
| 327 | func dbVersionHeaders(w http.ResponseWriter, r *http.Request, total, offset, size int64) error { |
| 328 | n, err := dbVersionRange(total, offset, size) |
| 329 | if err != nil { |
| 330 | return err |
| 331 | } |
| 332 | w.Header().Set("Content-Length", strconv.FormatInt(n, 10)) |
| 333 | w.Header().Set(dbdoor.HandleHeader, r.Header.Get(dbdoor.HandleHeader)) |
| 334 | if r.Header.Get("Range") != "" && n > 0 { |
| 335 | w.Header().Set("Content-Range", fmt.Sprintf("bytes %d-%d/%d", offset, offset+n-1, total)) |
| 336 | } |
| 337 | return nil |
| 338 | } |
| 339 | |
amadan