| 1 | // The forwarded `carlos db` read (db region forward spec |
| 2 | // docs/superpowers/specs/2026-10-22-db-region-forward-design.md): the list |
| 3 | // door's marker for a cross-region instance, and the one route that admits a |
| 4 | // read, queues it for a box in the data's region, and relays the box's sealed |
| 5 | // answer inside the same request. |
| 6 | package console |
| 7 | |
| 8 | import ( |
| 9 | "context" |
| 10 | "crypto/ed25519" |
| 11 | "crypto/sha256" |
| 12 | "encoding/hex" |
| 13 | "encoding/json" |
| 14 | "errors" |
| 15 | "fmt" |
| 16 | "io" |
| 17 | "log" |
| 18 | "net/http" |
| 19 | "os" |
| 20 | "slices" |
| 21 | "strings" |
| 22 | "time" |
| 23 | "unicode/utf8" |
| 24 | |
| 25 | "github.com/carlosframework/platform/internal/dbdoor" |
| 26 | "github.com/carlosframework/platform/internal/dbquery" |
| 27 | "github.com/carlosframework/platform/internal/dbreads" |
| 28 | "github.com/carlosframework/platform/internal/hostclaim" |
| 29 | "github.com/carlosframework/platform/internal/provision" |
| 30 | "github.com/carlosframework/platform/internal/registry" |
| 31 | "github.com/carlosframework/platform/internal/release" |
| 32 | ) |
| 33 | |
| 34 | // dbForwardTimes is the forwarded read's timing (spec §5.2, §5.5). Tests |
| 35 | // shrink it; production uses defaultDBForwardTimes. |
| 36 | type dbForwardTimes struct { |
| 37 | poll time.Duration // how often the console reads picked, result and chunks |
| 38 | wait time.Duration // a progress line at least this often |
| 39 | stall time.Duration // a write or flush that blocks this long ends the request |
| 40 | preHeader time.Duration // everything before the 200 (backhaul waits 30 s for headers) |
| 41 | pickup time.Duration // picked must appear this long after issued_at |
| 42 | grace time.Duration // after the owner's deadline |
| 43 | bound time.Duration // the request's own bound after issued_at |
| 44 | call time.Duration // one store call inside the stream (plan review finding 12) |
| 45 | |
| 46 | // beforeWrite, nil in production, runs just before admission checks the |
| 47 | // request's context ahead of each write ("audit", "request", "queue", |
| 48 | // "serial"): a test ends the request there (plan review finding 28). |
| 49 | beforeWrite func(step string) |
| 50 | } |
| 51 | |
| 52 | func defaultDBForwardTimes() dbForwardTimes { |
| 53 | return dbForwardTimes{ |
| 54 | poll: time.Second, wait: 15 * time.Second, stall: 30 * time.Second, preHeader: 20 * time.Second, |
| 55 | pickup: dbreads.ConsoleWaitsPicked, grace: dbreads.OwnerGrace, bound: dbreads.RequestBound, |
| 56 | call: 5 * time.Second, |
| 57 | } |
| 58 | } |
| 59 | |
| 60 | // dbForwardSigner is the root signer when this console forwards: the switch |
| 61 | // on, no host claims (a host-claim console answers with the residency |
| 62 | // sentence, as it refuses carlos run), and a key whose public half is the |
| 63 | // deployment root a box verifies against. A console holding only a scoped |
| 64 | // key cannot sign a request, and so cannot forward (spec §5.5). |
| 65 | func (s *Server) dbForwardSigner() (release.Signer, bool) { |
| 66 | if !s.cfg.DBForward || s.cfg.HostClaim || s.cfg.Key == nil { |
| 67 | return release.Signer{}, false |
| 68 | } |
| 69 | pub, ok := s.cfg.Key.Public().(ed25519.PublicKey) |
| 70 | if !ok || !pub.Equal(s.cfg.ReleasePub) { |
| 71 | return release.Signer{}, false |
| 72 | } |
| 73 | return release.RootSigner(s.cfg.Key), true |
| 74 | } |
| 75 | |
| 76 | const ( |
| 77 | dbForwardNone = "none" // DBF-1: no capable, non-draining box publishes in the region |
| 78 | dbForwardDead = "dead" // DBF-2: capable boxes, none live |
| 79 | ) |
| 80 | |
| 81 | // dbForwardEvidence is spec §7.1's rule. It reads Complete and |
| 82 | // BoxesComplete, never the error alone: ReadLiveSnapshot answers a nil |
| 83 | // error when status snapshots cannot be loaded and reports the gap in those |
| 84 | // flags (codex 10). Incomplete evidence comes first and wins over both |
| 85 | // refusals: the request is written and the pickup window decides. |
| 86 | func dbForwardEvidence(snap edgesSnapshot, err error, region string) string { |
| 87 | if err != nil || !snap.Complete || !snap.BoxesComplete { |
| 88 | return "" |
| 89 | } |
| 90 | capable, live := 0, 0 |
| 91 | for _, f := range snap.Files { |
| 92 | if f.Placement != region || f.Draining || !slices.Contains(f.Capabilities, dbreads.Capability) { |
| 93 | continue |
| 94 | } |
| 95 | capable++ |
| 96 | if f.Fresh { |
| 97 | live++ |
| 98 | } |
| 99 | } |
| 100 | switch { |
| 101 | case capable == 0: |
| 102 | return dbForwardNone |
| 103 | case live == 0: |
| 104 | return dbForwardDead |
| 105 | } |
| 106 | return "" |
| 107 | } |
| 108 | |
| 109 | // The console's forwarded-read sentences (copy record DBF-1, DBF-2, DBF-14), |
| 110 | // approved by Luna on 2026-10-23. |
| 111 | const ( |
| 112 | dbfNoCapable = "%s keeps its data in region %s. no server in that region can read databases yet, so carlos db cannot read it for now." |
| 113 | dbfNoneLive = "%s keeps its data in region %s. no server in that region is answering right now. try again in a few minutes." |
| 114 | dbfUpdateCarlos = "%s keeps its data in region %s. this version of carlos cannot read it from there. run carlos update, then try again." |
| 115 | ) |
| 116 | |
| 117 | // forwardable is whether this console forwards reads of rec: it can sign, |
| 118 | // and rec is in a named platform region (a fleet placement keeps its own |
| 119 | // sentence, and a home-region record seen from a regional console has no |
| 120 | // region to name). |
| 121 | func (s *Server) forwardable(rec provision.Record) bool { |
| 122 | _, fleet := registry.FleetPlacement(rec.Region) |
| 123 | _, ok := s.dbForwardSigner() |
| 124 | return ok && !fleet && dbreads.ValidRegion(rec.Region) |
| 125 | } |
| 126 | |
| 127 | // dbResidency answers a record homed outside this console's region (spec |
| 128 | // §5.5 step 1, §9.2). Not forwardable: main's refusal, byte for byte. |
| 129 | // Forwardable: DBF-1 or DBF-2 on complete evidence, else the forward marker, |
| 130 | // whose sentence an old CLI prints (DBF-14). |
| 131 | func (s *Server) dbResidency(w http.ResponseWriter, r *http.Request, rec provision.Record) { |
| 132 | if !s.forwardable(rec) { |
| 133 | jsonError(w, http.StatusConflict, s.dbResidencyRefusal(rec)) |
| 134 | return |
| 135 | } |
| 136 | snap, err := s.liveEdges(r.Context()) |
| 137 | switch dbForwardEvidence(snap, err, rec.Region) { |
| 138 | case dbForwardNone: |
| 139 | jsonError(w, http.StatusConflict, fmt.Sprintf(dbfNoCapable, rec.Host, rec.Region)) |
| 140 | return |
| 141 | case dbForwardDead: |
| 142 | jsonError(w, http.StatusConflict, fmt.Sprintf(dbfNoneLive, rec.Host, rec.Region)) |
| 143 | return |
| 144 | } |
| 145 | writeJSON(w, http.StatusConflict, dbdoor.ForwardBody{ |
| 146 | Error: fmt.Sprintf(dbfUpdateCarlos, rec.Host, rec.Region), Forward: 1, |
| 147 | Region: rec.Region, ForwardOps: []string{dbreads.OpQuery, dbreads.OpPull}, |
| 148 | }) |
| 149 | } |
| 150 | |
| 151 | // The forwarded read's other console sentences (copy record DBF-9, DBF-10, |
| 152 | // DBF-21, DBF-22), approved by Luna on 2026-10-23. |
| 153 | const ( |
| 154 | dbfTooMany = "too many database reads are already running. wait a minute and try again." |
| 155 | dbfMoving = "%s is moving between regions. read its database once the move has finished." |
| 156 | dbfAudit = "forwarded a database query to region %s. the query was %d bytes, sha256 %s (cli)" |
| 157 | dbfAuditAt = "forwarded a database query to region %s, as of %s. the query was %d bytes, sha256 %s (cli)" |
| 158 | ) |
| 159 | |
| 160 | // The pull's audit details (copy record DBF-19, DBF-20), approved by Luna |
| 161 | // on 2026-10-23. |
| 162 | const ( |
| 163 | dbfAuditPull = "forwarded a database pull to region %s (cli)" |
| 164 | dbfAuditPullAt = "forwarded a database pull to region %s, as of %s (cli)" |
| 165 | ) |
| 166 | |
| 167 | const dbReadRetryAfter = "30" |
| 168 | |
| 169 | // dbForwardAuditDetail is the db-read row's detail (spec §9.3): for a query |
| 170 | // the SQL's length and the first 16 hex digits of its SHA-256, never its |
| 171 | // text; a pull carries no SQL. |
| 172 | func dbForwardAuditDetail(op, region, sqlText string, at time.Time) string { |
| 173 | if op == dbreads.OpPull { |
| 174 | if at.IsZero() { |
| 175 | return fmt.Sprintf(dbfAuditPull, region) |
| 176 | } |
| 177 | return fmt.Sprintf(dbfAuditPullAt, region, at.UTC().Format(time.RFC3339)) |
| 178 | } |
| 179 | sum := sha256.Sum256([]byte(sqlText)) |
| 180 | sha := hex.EncodeToString(sum[:])[:16] |
| 181 | if at.IsZero() { |
| 182 | return fmt.Sprintf(dbfAudit, region, len(sqlText), sha) |
| 183 | } |
| 184 | return fmt.Sprintf(dbfAuditAt, region, at.UTC().Format(time.RFC3339), len(sqlText), sha) |
| 185 | } |
| 186 | |
| 187 | // dbReadAdmitted is a read this console has admitted, signed and queued. |
| 188 | type dbReadAdmitted struct { |
| 189 | id, region string |
| 190 | issued time.Time |
| 191 | } |
| 192 | |
| 193 | // handleAPIDBRead is POST /api/cli/apps/{account}/{app}/db/{host}/reads |
| 194 | // (spec §9.1): one forwarded read, which exists only inside this request |
| 195 | // (§5.2, J1). The admission counts are released by this handler's return and |
| 196 | // nowhere else (J5). |
| 197 | func (s *Server) handleAPIDBRead(w http.ResponseWriter, r *http.Request) { |
| 198 | rc := http.NewResponseController(w) |
| 199 | // Everything before the 200, the body included, gets s.dbf.preHeader: |
| 200 | // a body delivered slowly must not hold the request past backhaul's |
| 201 | // 30 s header wait (plan review finding 13). |
| 202 | if err := rc.SetReadDeadline(time.Now().Add(s.dbf.preHeader)); err != nil && !errors.Is(err, http.ErrNotSupported) { |
| 203 | log.Printf("console: db read: set the admission read deadline: %v", err) |
| 204 | } |
| 205 | a, actor, ok := s.admitDBRead(w, r) |
| 206 | if !ok { |
| 207 | return |
| 208 | } |
| 209 | defer s.dbReads.release(actor) |
| 210 | // Then the server's ReadTimeout would end this connection under a read |
| 211 | // that lasts longer; the read's own bound holds it instead, as the |
| 212 | // import door's read deadline does (apiinstanceimport.go). |
| 213 | if err := rc.SetReadDeadline(time.Now().Add(s.dbf.bound + time.Minute)); err != nil && !errors.Is(err, http.ErrNotSupported) { |
| 214 | log.Printf("console: db read: extend the read deadline: %v", err) |
| 215 | } |
| 216 | s.streamDBRead(w, r, rc, a) |
| 217 | } |
| 218 | |
| 219 | // readDBReadBody reads the whole body, to EOF, under the admission read |
| 220 | // deadline handleAPIDBRead set, and requires it to be exactly one JSON value |
| 221 | // (Astra round 1, P1). readJSONLimit decodes one value and leaves any tail |
| 222 | // unread; net/http would drain that tail when the response is first |
| 223 | // flushed, under the read deadline the stream extends to the read's bound, |
| 224 | // so a withheld tail could hold a request and its admission slot. The |
| 225 | // refusals are readJSONLimit's own sentences. |
| 226 | func readDBReadBody(w http.ResponseWriter, r *http.Request, out *dbreads.ReadBody) bool { |
| 227 | ct, _, _ := strings.Cut(r.Header.Get("Content-Type"), ";") |
| 228 | if strings.TrimSpace(strings.ToLower(ct)) != "application/json" { |
| 229 | jsonError(w, http.StatusUnsupportedMediaType, "send a JSON body with Content-Type: application/json") |
| 230 | return false |
| 231 | } |
| 232 | raw, err := io.ReadAll(http.MaxBytesReader(w, r.Body, dbreads.MaxSQLBytes+8<<10)) |
| 233 | if err != nil { |
| 234 | var tooLarge *http.MaxBytesError |
| 235 | if errors.As(err, &tooLarge) { |
| 236 | jsonError(w, http.StatusRequestEntityTooLarge, "request body too large") |
| 237 | return false |
| 238 | } |
| 239 | jsonError(w, http.StatusBadRequest, "malformed JSON body") |
| 240 | return false |
| 241 | } |
| 242 | // json.Unmarshal refuses anything but whitespace after the one value. |
| 243 | if json.Unmarshal(raw, out) != nil { |
| 244 | jsonError(w, http.StatusBadRequest, "malformed JSON body") |
| 245 | return false |
| 246 | } |
| 247 | return true |
| 248 | } |
| 249 | |
| 250 | // cancelDBRead is the one write J1 allows after a read's request ends |
| 251 | // without a result: best effort, so the owner stops early and a box that |
| 252 | // has not picked it skips it. |
| 253 | func (s *Server) cancelDBRead(id string) { |
| 254 | ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
| 255 | defer cancel() |
| 256 | if _, err := s.cfg.Releases.Put(ctx, dbreads.CancelKey(id), nil, ""); err != nil && !errors.Is(err, release.ErrCASConflict) { |
| 257 | log.Printf("console: db read %s: cancel: %v", id, err) |
| 258 | } |
| 259 | } |
| 260 | |
| 261 | // announceDBRead bumps the serial on the request's own context, so a |
| 262 | // request that has already ended announces nothing (J1; plan review finding |
| 263 | // 11). bumpSerial detaches on purpose, for callers whose writes outlive |
| 264 | // them; this read does not outlive its request. |
| 265 | func (s *Server) announceDBRead(ctx context.Context) { |
| 266 | ctx, cancel := context.WithTimeout(ctx, 2*time.Second) |
| 267 | defer cancel() |
| 268 | if err := release.BumpSerial(ctx, s.cfg.Releases, time.Now()); err != nil && ctx.Err() == nil { |
| 269 | log.Printf("console: db read: serial bump: %v (boxes list every 30 s anyway)", err) |
| 270 | } |
| 271 | } |
| 272 | |
| 273 | // admitDBRead is spec §5.5 step 3, all of it bounded by s.dbf.preHeader so |
| 274 | // the 200 leaves well inside backhaul's 30 s header wait: dbGate's order |
| 275 | // (person, membership with the byte-identical 404, the db budget, |
| 276 | // configuration, the record, the class rail, pooled, fleet), then a moving |
| 277 | // record, the SQL guard, capability on complete evidence, the admission |
| 278 | // counts, and the audit row, the signed request, its queue entry and the |
| 279 | // serial. Any refusal is an ordinary status and JSON error. |
| 280 | func (s *Server) admitDBRead(w http.ResponseWriter, r *http.Request) (dbReadAdmitted, string, bool) { |
| 281 | none := dbReadAdmitted{} |
| 282 | ctx, cancel := context.WithTimeout(r.Context(), s.dbf.preHeader) |
| 283 | defer cancel() |
| 284 | if p, ok := principalFrom(r.Context()); !ok || p.Kind != PrincipalUser { |
| 285 | http.NotFound(w, r) |
| 286 | return none, "", false |
| 287 | } |
| 288 | actor, acct, app, ok := s.apiAccountApp(w, r) |
| 289 | if !ok { |
| 290 | return none, "", false |
| 291 | } |
| 292 | if !s.cli.db.allow(hashToken(bearer(r)), time.Now()) { |
| 293 | jsonError(w, http.StatusTooManyRequests, "too many database requests. Wait a minute and try again.") |
| 294 | return none, "", false |
| 295 | } |
| 296 | if s.cfg.Releases == nil || s.cfg.HibernateBucket == "" { |
| 297 | jsonError(w, http.StatusServiceUnavailable, "this console has no data bucket set up, so it cannot read databases.") |
| 298 | return none, "", false |
| 299 | } |
| 300 | var body dbreads.ReadBody |
| 301 | if !readDBReadBody(w, r, &body) { |
| 302 | return none, "", false |
| 303 | } |
| 304 | host := r.PathValue("host") |
| 305 | src, err := s.resolveDBSource(ctx, acct, actor, app, host) |
| 306 | switch { |
| 307 | case errors.Is(err, os.ErrNotExist), errors.Is(err, ErrBadHost): |
| 308 | http.NotFound(w, r) |
| 309 | return none, "", false |
| 310 | case errors.Is(err, errSourceClass): |
| 311 | jsonError(w, http.StatusConflict, dbClassRefusal(src.rec)) |
| 312 | return none, "", false |
| 313 | case errors.Is(err, ErrPooled): |
| 314 | jsonError(w, http.StatusConflict, fmt.Sprintf( |
| 315 | "%s runs in the pool %s, so its data is on the pool's own server. carlos db cannot read it.", host, src.rec.Pool)) |
| 316 | return none, "", false |
| 317 | case errors.Is(err, ErrResidency): |
| 318 | // The case this route exists for. |
| 319 | case err == nil: |
| 320 | // In region: the db door serves it directly. There is no forwarded |
| 321 | // read for it until in-region reads move to the box path (plan 4). |
| 322 | http.NotFound(w, r) |
| 323 | return none, "", false |
| 324 | case errors.Is(err, errSourceResolve): |
| 325 | log.Printf("console: db read %s/%s/%s: resolve: %v", acct.Sqid, app, host, err) |
| 326 | jsonError(w, http.StatusBadGateway, "could not read the replica. Try again.") |
| 327 | return none, "", false |
| 328 | default: |
| 329 | log.Printf("console: db read %s/%s/%s: read record: %v", acct.Sqid, app, host, err) |
| 330 | jsonError(w, http.StatusBadGateway, "could not read the instance. Try again.") |
| 331 | return none, "", false |
| 332 | } |
| 333 | rec := src.rec |
| 334 | signer, ok := s.dbForwardSigner() |
| 335 | if !ok || !s.forwardable(rec) { |
| 336 | jsonError(w, http.StatusConflict, s.dbResidencyRefusal(rec)) |
| 337 | return none, "", false |
| 338 | } |
| 339 | if rec.Move != nil { |
| 340 | jsonError(w, http.StatusConflict, fmt.Sprintf(dbfMoving, host)) |
| 341 | return none, "", false |
| 342 | } |
| 343 | switch body.Op { |
| 344 | case dbreads.OpQuery: |
| 345 | case dbreads.OpPull: |
| 346 | // A pull runs nothing: SQL in one is a request no CLI builds. |
| 347 | if body.SQL != "" { |
| 348 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) |
| 349 | return none, "", false |
| 350 | } |
| 351 | default: |
| 352 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) |
| 353 | return none, "", false |
| 354 | } |
| 355 | // encoding/json turns bytes that are not UTF-8, and a lone surrogate |
| 356 | // escape, into U+FFFD without a word, so the SQL decoded here may not be |
| 357 | // the SQL sent. The box must run exactly the bytes typed or be told no |
| 358 | // (plan 1's Review Focus 1), so any U+FFFD is refused; a query that |
| 359 | // meant one literally is refused too, and says so the same way (plan |
| 360 | // review finding 36). The CLI refuses first, with DBF-32. |
| 361 | if strings.ContainsRune(body.SQL, utf8.RuneError) { |
| 362 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) |
| 363 | return none, "", false |
| 364 | } |
| 365 | if body.Op == dbreads.OpQuery { |
| 366 | if err := dbquery.CheckSQL(body.SQL); err != nil || body.SQL == "" { |
| 367 | msg := http.StatusText(http.StatusBadRequest) |
| 368 | if err != nil { |
| 369 | msg = err.Error() |
| 370 | } |
| 371 | jsonError(w, http.StatusBadRequest, msg) |
| 372 | return none, "", false |
| 373 | } |
| 374 | } |
| 375 | var at time.Time |
| 376 | if body.At != "" { |
| 377 | if at, err = time.Parse(time.RFC3339, body.At); err != nil || at.After(time.Now().Add(dbreads.Skew)) { |
| 378 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) |
| 379 | return none, "", false |
| 380 | } |
| 381 | } |
| 382 | if _, err := dbreads.ParseReplyKey(body.ReplyKey); err != nil { |
| 383 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) |
| 384 | return none, "", false |
| 385 | } |
| 386 | snap, serr := s.liveEdges(ctx) |
| 387 | switch dbForwardEvidence(snap, serr, rec.Region) { |
| 388 | case dbForwardNone: |
| 389 | jsonError(w, http.StatusConflict, fmt.Sprintf(dbfNoCapable, host, rec.Region)) |
| 390 | return none, "", false |
| 391 | case dbForwardDead: |
| 392 | jsonError(w, http.StatusConflict, fmt.Sprintf(dbfNoneLive, host, rec.Region)) |
| 393 | return none, "", false |
| 394 | } |
| 395 | if !s.dbReads.acquire(actor) { |
| 396 | w.Header().Set("Retry-After", dbReadRetryAfter) |
| 397 | jsonError(w, http.StatusTooManyRequests, dbfTooMany) |
| 398 | return none, "", false |
| 399 | } |
| 400 | admitted := false |
| 401 | defer func() { |
| 402 | if !admitted { |
| 403 | s.dbReads.release(actor) |
| 404 | } |
| 405 | }() |
| 406 | fail := func(what string, err error) (dbReadAdmitted, string, bool) { |
| 407 | log.Printf("console: db read %s/%s/%s: %s: %v", acct.Sqid, app, host, what, err) |
| 408 | jsonError(w, http.StatusBadGateway, "could not read the instance. Try again.") |
| 409 | return none, "", false |
| 410 | } |
| 411 | id, err := dbreads.MintID() |
| 412 | if err != nil { |
| 413 | return fail("mint id", err) |
| 414 | } |
| 415 | inc, err := hostclaim.Incarnation(acct.Sqid, app, host, rec.CreatedAt) |
| 416 | if err != nil { |
| 417 | return fail("incarnation", err) |
| 418 | } |
| 419 | issued := time.Now().UTC().Truncate(time.Second) |
| 420 | atRaw := "" |
| 421 | if !at.IsZero() { |
| 422 | atRaw = at.UTC().Format(time.RFC3339Nano) |
| 423 | } |
| 424 | wire, err := dbreads.Sign(signer, dbreads.Request{ID: id, Account: acct.Sqid, App: app, Host: host, Incarnation: inc, |
| 425 | Region: rec.Region, Op: body.Op, SQL: body.SQL, At: atRaw, ReplyKey: body.ReplyKey, By: actor, |
| 426 | IssuedAt: issued.Format(time.RFC3339)}) |
| 427 | if err != nil { |
| 428 | // SQL the envelope cannot carry intact (invalid UTF-8). |
| 429 | http.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest) |
| 430 | return none, "", false |
| 431 | } |
| 432 | // J1 at every write (plan review finding 28): the request's context is |
| 433 | // checked immediately before each one, because the stores do not stop a |
| 434 | // write for it (DirStore.Put ignores its context). The audit row comes |
| 435 | // first, while the request is open and before any box can see the read, |
| 436 | // so a read a box runs always has its row (spec §9.3 rev 6). A request |
| 437 | // that ends before request.json is written leaves at most that row; one |
| 438 | // that ends after leaves request.json, perhaps the queue entry, and the |
| 439 | // cancel, never a serial bump. A row the store cannot write refuses the |
| 440 | // read rather than run it unaudited. |
| 441 | ended := func(step string, published bool) bool { |
| 442 | if s.dbf.beforeWrite != nil { |
| 443 | s.dbf.beforeWrite(step) |
| 444 | } |
| 445 | if ctx.Err() == nil { |
| 446 | return false |
| 447 | } |
| 448 | if published { |
| 449 | s.cancelDBRead(id) |
| 450 | } |
| 451 | return true |
| 452 | } |
| 453 | if ended("audit", false) { |
| 454 | return none, "", false |
| 455 | } |
| 456 | if err := s.store.RecordActivityContext(ctx, acct.ID, app, actor, Activity{Action: dbAuditAction, |
| 457 | TargetType: "instance", Target: host, Detail: dbForwardAuditDetail(body.Op, rec.Region, body.SQL, at)}, time.Now()); err != nil { |
| 458 | return fail("audit", err) |
| 459 | } |
| 460 | if ended("request", false) { |
| 461 | return none, "", false |
| 462 | } |
| 463 | if _, err := s.cfg.Releases.Put(ctx, dbreads.RequestKey(id), wire, ""); err != nil { |
| 464 | // An ambiguous create may have landed: cancel, as for the queue. |
| 465 | s.cancelDBRead(id) |
| 466 | return fail("write the request", err) |
| 467 | } |
| 468 | if ended("queue", true) { |
| 469 | return none, "", false |
| 470 | } |
| 471 | if _, err := s.cfg.Releases.Put(ctx, dbreads.QueueKey(rec.Region, issued, id), nil, ""); err != nil { |
| 472 | // The queue entry may have landed on an ambiguous answer: cancel, so |
| 473 | // a box that lists it skips it (J1's one write). |
| 474 | s.cancelDBRead(id) |
| 475 | return fail("write the queue entry", err) |
| 476 | } |
| 477 | if ended("serial", true) { |
| 478 | return none, "", false |
| 479 | } |
| 480 | // The bump runs on the request's context, not bumpSerial's detached one, |
| 481 | // so a request that ends while it runs does not announce (finding 33). |
| 482 | s.announceDBRead(ctx) |
| 483 | // The request may have ended during the bump (Astra round 1, P2): the |
| 484 | // read is published, so it gets its cancel, and the caller no 200. |
| 485 | if ctx.Err() != nil { |
| 486 | s.cancelDBRead(id) |
| 487 | return none, "", false |
| 488 | } |
| 489 | admitted = true |
| 490 | return dbReadAdmitted{id: id, region: rec.Region, issued: issued}, actor, true |
| 491 | } |
| 492 | |
| 493 | // errDBClientGone is a line the client did not take within s.dbf.stall. |
| 494 | var errDBClientGone = errors.New("the client went or stalled") |
| 495 | |
| 496 | // dbfEndSlack is how far past the request's bound the last line may still be |
| 497 | // written: room for the ended line, and no more. |
| 498 | const dbfEndSlack = 10 * time.Second |
| 499 | |
| 500 | // dbLineWriter writes one line at a time, each under a fresh write deadline |
| 501 | // and flushed under the same deadline (spec §5.5 step 6, codex 19). The |
| 502 | // deadline never passes until, the request's bound plus dbfEndSlack (plan |
| 503 | // review finding 12). A write or flush error ends the request as a stalled |
| 504 | // client. |
| 505 | type dbLineWriter struct { |
| 506 | w http.ResponseWriter |
| 507 | rc *http.ResponseController |
| 508 | stall time.Duration |
| 509 | until time.Time |
| 510 | last time.Time |
| 511 | } |
| 512 | |
| 513 | func (l *dbLineWriter) write(line dbreads.Line) error { |
| 514 | b, err := json.Marshal(line) |
| 515 | if err != nil { |
| 516 | return err |
| 517 | } |
| 518 | dl := time.Now().Add(l.stall) |
| 519 | if dl.After(l.until) { |
| 520 | dl = l.until |
| 521 | } |
| 522 | if err := l.rc.SetWriteDeadline(dl); err != nil && !errors.Is(err, http.ErrNotSupported) { |
| 523 | return err |
| 524 | } |
| 525 | if _, err := l.w.Write(append(b, '\n')); err != nil { |
| 526 | return err |
| 527 | } |
| 528 | if err := l.rc.Flush(); err != nil { |
| 529 | return err |
| 530 | } |
| 531 | l.last = time.Now() |
| 532 | return nil |
| 533 | } |
| 534 | |
| 535 | // tick waits one poll and writes a wait line when nothing has been written |
| 536 | // for every. It answers ctx's error when ctx ends first, and errDBClientGone |
| 537 | // when the line cannot be written. |
| 538 | func (l *dbLineWriter) tick(ctx context.Context, phase string, poll, every time.Duration) error { |
| 539 | select { |
| 540 | case <-ctx.Done(): |
| 541 | return ctx.Err() |
| 542 | case <-time.After(poll): |
| 543 | } |
| 544 | if time.Since(l.last) >= every && l.write(dbreads.Line{Wait: phase}) != nil { |
| 545 | return errDBClientGone |
| 546 | } |
| 547 | return nil |
| 548 | } |
| 549 | |
| 550 | // dbReadRelay is one admitted read's relay, with what its last log line |
| 551 | // counts. (dbStream is apidb.go's streaming function: plan review finding 25.) |
| 552 | type dbReadRelay struct { |
| 553 | s *Server |
| 554 | lw *dbLineWriter |
| 555 | chunks int |
| 556 | bytes int |
| 557 | } |
| 558 | |
| 559 | // dbfCall runs one store call under s.dbf.call and the phase's context, and |
| 560 | // keeps writing wait lines while it blocks, so a stalled store neither holds |
| 561 | // the request past its phase nor silences the stream (plan review finding |
| 562 | // 12). A store that ignores its context is left behind: its goroutine ends |
| 563 | // when the call does, into a buffered channel nobody reads. |
| 564 | func dbfCall[T any](st *dbReadRelay, ctx context.Context, phase string, op func(context.Context) (T, error)) (T, error) { |
| 565 | var zero T |
| 566 | cctx, cancel := context.WithTimeout(ctx, st.s.dbf.call) |
| 567 | defer cancel() |
| 568 | type outcome struct { |
| 569 | v T |
| 570 | err error |
| 571 | } |
| 572 | done := make(chan outcome, 1) |
| 573 | go func() { |
| 574 | v, err := op(cctx) |
| 575 | done <- outcome{v, err} |
| 576 | }() |
| 577 | t := time.NewTicker(st.s.dbf.poll) |
| 578 | defer t.Stop() |
| 579 | for { |
| 580 | select { |
| 581 | case o := <-done: |
| 582 | return o.v, o.err |
| 583 | case <-cctx.Done(): |
| 584 | return zero, cctx.Err() |
| 585 | case <-t.C: |
| 586 | if time.Since(st.lw.last) >= st.s.dbf.wait && st.lw.write(dbreads.Line{Wait: phase}) != nil { |
| 587 | return zero, errDBClientGone |
| 588 | } |
| 589 | } |
| 590 | } |
| 591 | } |
| 592 | |
| 593 | // readDBResult is result.json, or nil while there is none, and nil for one |
| 594 | // that is not a result in the vocabulary (Review Focus 4). |
| 595 | func (s *Server) readDBResult(ctx context.Context, id string) *dbreads.Result { |
| 596 | raw, _, err := s.cfg.Releases.Get(ctx, dbreads.ResultKey(id)) |
| 597 | if err != nil { |
| 598 | return nil |
| 599 | } |
| 600 | var res dbreads.Result |
| 601 | if json.Unmarshal(raw, &res) != nil || res.Valid() != nil { |
| 602 | return nil |
| 603 | } |
| 604 | return &res |
| 605 | } |
| 606 | |
| 607 | // streamDBRead is spec §5.5 steps 6 and 7 inside the admitted request: |
| 608 | // admitted at once, picked once, then about once a second result.json first |
| 609 | // and the chunk listing second (codex 20; the run follow door's order), each |
| 610 | // chunk in sequence with no gaps, a wait line whenever s.dbf.wait passes |
| 611 | // with nothing else, and the result last, only once every chunk through its |
| 612 | // last_seq has been written. Each phase runs on its own deadline context: |
| 613 | // pickup on issued_at plus s.dbf.pickup, the owner's on its deadline plus |
| 614 | // s.dbf.grace, never later than s.dbf.bound after issued_at. It ends without |
| 615 | // a result, and writes cancel, when either deadline passes, a result |
| 616 | // included, or when the client goes or stalls. Its last log line is the |
| 617 | // relay's own measure (plan review finding 21). |
| 618 | func (s *Server) streamDBRead(w http.ResponseWriter, r *http.Request, rc *http.ResponseController, a dbReadAdmitted) { |
| 619 | ctx := r.Context() |
| 620 | bound := a.issued.Add(s.dbf.bound) |
| 621 | w.Header().Set("Content-Type", "application/x-ndjson") |
| 622 | w.Header().Set("Cache-Control", "no-store") |
| 623 | w.WriteHeader(http.StatusOK) |
| 624 | st := &dbReadRelay{s: s, lw: &dbLineWriter{w: w, rc: rc, stall: s.dbf.stall, until: bound.Add(dbfEndSlack)}} |
| 625 | start, why := time.Now(), "client gone" |
| 626 | defer func() { |
| 627 | log.Printf("console: db read %s: relayed %d chunks (%d bytes) in %s, ended %s", |
| 628 | a.id, st.chunks, st.bytes, time.Since(start).Round(time.Millisecond), why) |
| 629 | }() |
| 630 | end := func(ended string) { |
| 631 | if ended != "" { |
| 632 | why = ended |
| 633 | st.lw.write(dbreads.Line{Ended: ended}) |
| 634 | } |
| 635 | s.cancelDBRead(a.id) |
| 636 | } |
| 637 | // stop ends a phase: a client that went or stalled gets no line, a phase |
| 638 | // whose deadline passed gets its ended word. |
| 639 | stop := func(err error, ended string) { |
| 640 | if errors.Is(err, errDBClientGone) || ctx.Err() != nil { |
| 641 | end("") |
| 642 | return |
| 643 | } |
| 644 | end(ended) |
| 645 | } |
| 646 | if st.lw.write(dbreads.Line{Admitted: a.id}) != nil { |
| 647 | end("") |
| 648 | return |
| 649 | } |
| 650 | store := s.cfg.Releases |
| 651 | |
| 652 | pickCtx, cancelPick := context.WithDeadline(ctx, a.issued.Add(s.dbf.pickup)) |
| 653 | defer cancelPick() |
| 654 | var picked dbreads.Picked |
| 655 | for { |
| 656 | raw, err := dbfCall(st, pickCtx, dbreads.WaitPickup, func(c context.Context) ([]byte, error) { |
| 657 | b, _, err := store.Get(c, dbreads.PickedKey(a.id)) |
| 658 | return b, err |
| 659 | }) |
| 660 | if err == nil && json.Unmarshal(raw, &picked) == nil { |
| 661 | break |
| 662 | } |
| 663 | if pickCtx.Err() != nil || errors.Is(err, errDBClientGone) { |
| 664 | stop(err, dbreads.EndedNotPicked) |
| 665 | return |
| 666 | } |
| 667 | if err := st.lw.tick(pickCtx, dbreads.WaitPickup, s.dbf.poll, s.dbf.wait); err != nil { |
| 668 | stop(err, dbreads.EndedNotPicked) |
| 669 | return |
| 670 | } |
| 671 | } |
| 672 | if st.lw.write(dbreads.Line{Picked: picked.Box}) != nil { |
| 673 | end("") |
| 674 | return |
| 675 | } |
| 676 | deadline := picked.Deadline.Add(s.dbf.grace) |
| 677 | if deadline.After(bound) { |
| 678 | deadline = bound |
| 679 | } |
| 680 | ownCtx, cancelOwn := context.WithDeadline(ctx, deadline) |
| 681 | defer cancelOwn() |
| 682 | next := int64(1) |
| 683 | for { |
| 684 | res, err := dbfCall(st, ownCtx, dbreads.WaitReading, func(c context.Context) (*dbreads.Result, error) { |
| 685 | return s.readDBResult(c, a.id), nil |
| 686 | }) // first |
| 687 | if ownCtx.Err() != nil || errors.Is(err, errDBClientGone) { |
| 688 | stop(err, dbreads.EndedNoResult) |
| 689 | return |
| 690 | } |
| 691 | keys, err := dbfCall(st, ownCtx, dbreads.WaitReading, func(c context.Context) ([]string, error) { |
| 692 | return store.List(c, dbreads.OutPrefix(a.id)) |
| 693 | }) |
| 694 | if ownCtx.Err() != nil || errors.Is(err, errDBClientGone) { |
| 695 | stop(err, dbreads.EndedNoResult) |
| 696 | return |
| 697 | } |
| 698 | if err == nil { |
| 699 | listed := map[int64]bool{} |
| 700 | for _, k := range keys { |
| 701 | if seq, ok := dbreads.ParseChunkKey(k, dbreads.OutPrefix(a.id)); ok { |
| 702 | listed[seq] = true |
| 703 | } |
| 704 | } |
| 705 | for listed[next] { |
| 706 | seq := next |
| 707 | sealed, err := dbfCall(st, ownCtx, dbreads.WaitReading, func(c context.Context) ([]byte, error) { |
| 708 | b, _, err := store.Get(c, dbreads.ChunkKey(a.id, seq)) |
| 709 | return b, err |
| 710 | }) |
| 711 | if ownCtx.Err() != nil || errors.Is(err, errDBClientGone) { |
| 712 | stop(err, dbreads.EndedNoResult) |
| 713 | return |
| 714 | } |
| 715 | if err != nil { |
| 716 | break |
| 717 | } |
| 718 | if st.lw.write(dbreads.Line{Chunk: seq, Data: sealed}) != nil { |
| 719 | end("") |
| 720 | return |
| 721 | } |
| 722 | st.chunks++ |
| 723 | st.bytes += len(sealed) |
| 724 | next++ |
| 725 | } |
| 726 | } |
| 727 | if res != nil && next > res.LastSeq { |
| 728 | // The bound holds for a result too: one that is complete only |
| 729 | // after the deadline is not relayed (plan review finding 12). |
| 730 | if ownCtx.Err() != nil { |
| 731 | stop(nil, dbreads.EndedNoResult) |
| 732 | return |
| 733 | } |
| 734 | why = res.State |
| 735 | st.lw.write(dbreads.Line{Result: res}) |
| 736 | return |
| 737 | } |
| 738 | if err := st.lw.tick(ownCtx, dbreads.WaitReading, s.dbf.poll, s.dbf.wait); err != nil { |
| 739 | stop(err, dbreads.EndedNoResult) |
| 740 | return |
| 741 | } |
| 742 | } |
| 743 | } |
| 744 | |
amadan