amadan

carlos / platform Public

carlos db pull and query: read an instance's database from its replica (spec 2026-09-02, unit 1)

Squashed from feat/carlos-db-read into main.
Co-Authored-By: Paul Campbell <paul@tito.io>
claude da866a929171673a29ad99004dc07d53e42f9d4a parent 7edd4b3
28 files changed, +6205 −10
  • apimode.go +12 −0
  • auth.go +38 −0
  • cmd/console/main.go +31 −2
  • cmd/console/wiring_test.go +32 −0
  • db.go +620 −0
  • db_e2e_test.go +249 −0
  • db_test.go +577 −0
  • docs/site/cli.md +30 −0
  • docs/superpowers/plans/2026-09-05-carlos-db-unit-1.md +828 −0
  • docs/superpowers/specs/2026-09-02-carlos-db-design.md +646 −0
  • internal/console/apidb.go +471 −0
  • internal/console/apidb_test.go +862 −0
  • internal/console/authcli.go +13 −0
  • internal/console/server.go +28 −0
  • internal/console/svcroutes_test.go +11 −2
  • internal/dbdoor/client.go +157 −0
  • internal/dbdoor/client_test.go +312 −0
  • internal/dbdoor/coords.go +133 −0
  • internal/dbdoor/coords_test.go +77 −0
  • internal/dbdoor/dbdoortest/replica.go +78 −0
  • internal/dbdoor/dbdoortest/replica_test.go +49 −0
  • internal/dbdoor/restore.go +340 −0
  • internal/dbdoor/restore_test.go +523 −0
  • internal/lease/check.go +13 −6
  • internal/lease/check_test.go +15 −0
  • internal/lease/endpoint.go +19 −0
  • internal/lease/endpoint_test.go +37 −0
  • main.go +4 −0

CI passed — run details

diff --git a/apimode.go b/apimode.go
index 1e13c746e..85c63470d 100644
--- a/apimode.go
+++ b/apimode.go
@@ -37,6 +37,7 @@ import (
"github.com/carlosframework/platform/internal/apperr"
"github.com/carlosframework/platform/internal/applog"
"github.com/carlosframework/platform/internal/caps"
+ "github.com/carlosframework/platform/internal/dbdoor"
"github.com/carlosframework/platform/internal/ledger"
"github.com/carlosframework/platform/internal/objstore"
"github.com/carlosframework/platform/internal/schedule"
@@ -1055,6 +1056,17 @@ type apiRestartResult struct {
App string `json:"app"`
}
+// dbClient is the `carlos db` replica client for one instance (db.go): a
+// read-only litestream.ReplicaClient whose every request is a GET through
+// this session's console, as this session's person. The account segment
+// and its ?exact=sqid switch ride on apiPath, exactly as every other verb's
+// do; the client itself only ever names a coordinate.
+func (s *apiSession) dbClient(a apiAcct, app, host string) *dbdoor.Client {
+ return dbdoor.NewClient(func(ctx context.Context, tail string, h http.Header) (*http.Response, error) {
+ return s.c.doStream(ctx, http.MethodGet, apiPath(a, app, "/db/"+url.PathEscape(host)+tail), nil, -1, h)
+ })
+}
+
// restart asks the console to request a process cycle for app on every box
// that serves it (issue #113). Mirrors trashApp's shape — postJSON, a 200
// decodes, anything else is apiErrorFromBody's job — with ONE addition, the
diff --git a/auth.go b/auth.go
index 3e282cf12..e0b29a9d8 100644
--- a/auth.go
+++ b/auth.go
@@ -535,6 +535,44 @@ func (c *consoleClient) doMultipartMethod(ctx context.Context, method, path stri
return res.StatusCode, body, nil
}
+// doStream runs one request whose response body is a STREAM — `carlos db`
+// reading a replica file down — and hands the response back unread, so the
+// caller copies the body where it is going without holding it whole.
+// header carries the caller's extras (a Range) on top of the bearer.
+//
+// Same client shape as doMultipart, for the same reason: no absolute
+// Timeout (the caller's context is the budget, and a large file over a slow
+// link outlives the JSON client's 30s), and no redirects. A non-2xx status
+// is NOT an error here: the caller reads the body itself. The caller closes
+// the response body.
+func (c *consoleClient) doStream(ctx context.Context, method, path string, body io.Reader, contentLength int64, header http.Header) (*http.Response, error) {
+ req, err := http.NewRequestWithContext(ctx, method, c.base+path, body)
+ if err != nil {
+ return nil, err
+ }
+ if body != nil && contentLength >= 0 {
+ req.ContentLength = contentLength
+ }
+ for k, vs := range header {
+ for _, v := range vs {
+ req.Header.Add(k, v)
+ }
+ }
+ if c.token != "" {
+ req.Header.Set("Authorization", "Bearer "+c.token)
+ }
+ client := &http.Client{Timeout: shipHTTPClientTimeout, CheckRedirect: neverFollow}
+ res, err := client.Do(req)
+ if err != nil {
+ return nil, err
+ }
+ if merr := consoleMoved(c.base, res.StatusCode, res.Header.Get("Location")); merr != nil {
+ res.Body.Close()
+ return nil, offerConsoleMoved(merr)
+ }
+ return res, nil
+}
+
// neverFollow is the one redirect policy every console client uses.
func neverFollow(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
diff --git a/cmd/console/main.go b/cmd/console/main.go
index 8b5a8e789..4a3b04776 100644
--- a/cmd/console/main.go
+++ b/cmd/console/main.go
@@ -82,6 +82,7 @@ import (
"strings"
"time"
+ "github.com/carlosframework/platform/internal/activator"
"github.com/carlosframework/platform/internal/appconfig"
"github.com/carlosframework/platform/internal/console"
"github.com/carlosframework/platform/internal/dnsrec"
@@ -265,6 +266,7 @@ func serve() {
// One release.Store, named, because the senders sweep below needs the
// same one the server got.
rel := openStore()
+ dbBucket, dbRegion, dbPrefix, placement := dbDoorWiring()
srv, err := console.NewServer(console.Config{
Origin: origin,
Keymail: envOr("CONSOLE_KEYMAIL", "https://keymail.dev"),
@@ -274,8 +276,14 @@ func serve() {
// opens it with — /fleet/architecture draws the box -> bucket edge
// from it. Empty on a CARLOS_DEPLOYMENT_DIR deployment, which
// correctly draws no bucket node.
- Bucket: os.Getenv("CARLOS_DEPLOYMENT_BUCKET"),
- HomeRegion: strings.TrimSpace(os.Getenv("CONSOLE_HOME_REGION")),
+ Bucket: os.Getenv("CARLOS_DEPLOYMENT_BUCKET"),
+ HomeRegion: strings.TrimSpace(os.Getenv("CONSOLE_HOME_REGION")),
+ // The carlos db door's data bucket and this box's placement
+ // (dbDoorWiring).
+ HibernateBucket: dbBucket,
+ HibernateRegion: dbRegion,
+ HibernatePrefix: dbPrefix,
+ Region: placement,
ReleasePub: pub,
Key: key,
EdgeIPs: edgeIPs(),
@@ -1046,3 +1054,24 @@ func envOr(k, def string) string {
}
return def
}
+
+// dbDoorWiring reads what the `carlos db` door needs (internal/console
+// apidb.go; spec 2026-09-02 §6): the region data bucket triple, parsed by
+// the activator's own ConfigFromEnv so the console reads exactly the
+// prefixes the agent on this box writes, plus CARLOS_REGION, this box's
+// placement, which an instance record's region must equal. An empty bucket
+// leaves the door answering 503. A config the agent itself would refuse
+// turns the door off rather than failing console boot: the console's job is
+// not to police the agent's idle timer.
+func dbDoorWiring() (bucket, region, prefix, placement string) {
+ placement = strings.TrimSpace(os.Getenv("CARLOS_REGION"))
+ cfg, err := activator.ConfigFromEnv()
+ if err != nil {
+ log.Printf("console: %v; the carlos db door is off", err)
+ return "", "", "", placement
+ }
+ if cfg == nil {
+ return "", "", "", placement
+ }
+ return cfg.Bucket, cfg.Region, cfg.Prefix, placement
+}
diff --git a/cmd/console/wiring_test.go b/cmd/console/wiring_test.go
index 531256af5..415e59a8f 100644
--- a/cmd/console/wiring_test.go
+++ b/cmd/console/wiring_test.go
@@ -64,3 +64,35 @@ func TestScannerWiring(t *testing.T) {
}
})
}
+
+// The db door reads the agent's own CARLOS_HIBERNATE_* triple, parsed the
+// agent's way (trimmed, prefix slashes stripped), so the console reads
+// exactly the prefixes the activator beside it writes.
+func TestDBDoorWiringReadsTheAgentsTriple(t *testing.T) {
+ t.Setenv("CARLOS_HIBERNATE_BUCKET", " data-ie ")
+ t.Setenv("CARLOS_HIBERNATE_REGION", "eu-west-1")
+ t.Setenv("CARLOS_HIBERNATE_PREFIX", "/instances/")
+ t.Setenv("CARLOS_HIBERNATE_IDLE", "")
+ t.Setenv("CARLOS_REGION", " uk ")
+ bucket, region, prefix, placement := dbDoorWiring()
+ if bucket != "data-ie" || region != "eu-west-1" || prefix != "instances" || placement != "uk" {
+ t.Fatalf("= %q %q %q %q", bucket, region, prefix, placement)
+ }
+}
+
+// A bad idle duration is the agent's problem, not a reason to fail console
+// boot; it turns the door off instead, loudly.
+func TestDBDoorWiringIsOffOnABadAgentConfig(t *testing.T) {
+ t.Setenv("CARLOS_HIBERNATE_BUCKET", "data-ie")
+ t.Setenv("CARLOS_HIBERNATE_IDLE", "soon")
+ if bucket, _, _, _ := dbDoorWiring(); bucket != "" {
+ t.Fatalf("bucket = %q, want the door off", bucket)
+ }
+}
+
+func TestDBDoorWiringIsOffWithoutABucket(t *testing.T) {
+ t.Setenv("CARLOS_HIBERNATE_BUCKET", "")
+ if bucket, _, _, _ := dbDoorWiring(); bucket != "" {
+ t.Fatalf("bucket = %q, want the door off", bucket)
+ }
+}
diff --git a/db.go b/db.go
new file mode 100644
index 000000000..8efc220a4
--- /dev/null
+++ b/db.go
@@ -0,0 +1,620 @@
+// db.go is `carlos db`: a member reads an app's database from its replica,
+// with no box access and no AWS credentials (spec 2026-09-02).
+//
+// It never connects to the running database. The console serves the
+// replica's LTX files by coordinate (internal/console/apidb.go); this side
+// restores them, with litestream's own restore and a client that has no
+// write methods (internal/dbdoor), into a private 0700 directory, proves the
+// result with lease.CheckDB, and then either copies it to --out (pull) or
+// opens it read-only and runs one query (query). The console never executes
+// tenant SQL and never holds a restored database.
+//
+// Read-only is enforced, not assumed (spec §4): mode=ro and query_only. A
+// copy is harmless to write to; the point is that nobody, person or agent,
+// can come away believing a write reached production.
+//
+// Unit 1 of three (spec §13): ls and its freshness fields, then shell,
+// follow.
+package main
+
+import (
+ "context"
+ "database/sql"
+ "errors"
+ "flag"
+ "fmt"
+ "io"
+ "log"
+ "os"
+ "os/signal"
+ "sort"
+ "strings"
+ "syscall"
+ "time"
+
+ "modernc.org/sqlite"
+ sqlitelib "modernc.org/sqlite/lib"
+
+ "github.com/carlosframework/platform/internal/dbdoor"
+)
+
+const dbUsage = `usage: carlos db <verb> --app <a> [--instance <host>] [flags]
+
+ pull --out <path> restore a copy of the instance's database and save it
+ at <path>, which must not exist yet (saved 0600)
+ query --sql <text> restore a copy and run one read-only query against it;
+ rows go to stdout, tab-separated, with a header row
+
+Both read the platform's replica of the database, never the live one, and
+nothing you do to the copy reaches the instance. --instance names which of
+the app's instances to read; it may be left out when the app has one. The
+usual --account / --account-id / --console / --destination apply.
+`
+
+func runDB(args []string) {
+ if len(args) == 0 {
+ fmt.Fprint(os.Stderr, dbUsage)
+ os.Exit(2)
+ }
+ switch args[0] {
+ case "pull":
+ runDBPull(args[1:])
+ case "query":
+ runDBQuery(args[1:])
+ case "-h", "--help", "help":
+ fmt.Fprint(stdout, dbUsage)
+ default:
+ fmt.Fprintf(os.Stderr, "carlos db: unknown verb %q\n%s", args[0], dbUsage)
+ os.Exit(2)
+ }
+}
+
+// dbTarget is what both verbs resolve first: the session, the account, the
+// app and the one instance the verb reads.
+type dbTarget struct {
+ sess *apiSession
+ acct apiAcct
+ app string
+ host string
+}
+
+// dbFlags registers the flags both verbs share and returns the resolver.
+// Status lines go to stderr: query's stdout is its rows, and must pipe
+// cleanly into whatever reads them.
+func dbFlags(fs *flag.FlagSet, verb string) func() dbTarget {
+ app := appFlag(fs, "app whose database to read")
+ instance := fs.String("instance", "", "the instance whose database to read (may be left out when the app has one)")
+ acctRaw, acctIDRaw, _ := accountFlags(fs)
+ consoleFlag := consoleOption(fs, "console to read through (default: the only logged-in one)")
+ return func() dbTarget {
+ sess := requireDBAPISession(*consoleFlag)
+ printUsing(os.Stderr, sess)
+ acct := resolvedAccount(sess, acctRaw, acctIDRaw)
+ name := resolvedApp(sess, acct, strings.TrimSpace(*app), dbUsage)
+ host, err := resolveDBInstance(context.Background(), sess, acct, name, strings.TrimSpace(*instance))
+ if err != nil {
+ log.Fatalf("carlos db %s: %v", verb, err)
+ }
+ return dbTarget{sess: sess, acct: acct, app: name, host: host}
+ }
+}
+
+// requireDBAPISession resolves signed-in mode, refusing direct bucket mode
+// by name: the door is the console's, and an operator holding the bucket
+// already has a box.
+func requireDBAPISession(consoleFlag string) *apiSession {
+ sess, err := resolveMode(consoleFlag)
+ if err != nil {
+ log.Fatal(err)
+ }
+ if sess == nil {
+ log.Fatal("carlos db works through a console, and this shell is in direct bucket mode " +
+ "(CARLOS_DEPLOYMENT_BUCKET or CARLOS_DEPLOYMENT_DIR is set). Unset it and run: carlos auth login --console <url>.")
+ }
+ return sess
+}
+
+// resolveDBInstance is --instance, or the app's only instance when it has
+// exactly one. Several is a refusal naming them: guessing would hand
+// someone the wrong data while telling them it is the right data (spec §7).
+func resolveDBInstance(ctx context.Context, sess *apiSession, acct apiAcct, app, host string) (string, error) {
+ if host != "" {
+ return host, nil
+ }
+ recs, truncated, _, err := sess.instanceList(ctx, acct, app, "", "")
+ if err != nil {
+ return "", fmt.Errorf("could not list the app's instances to pick one: %w", err)
+ }
+ if len(recs) == 1 && !truncated {
+ return recs[0].Host, nil
+ }
+ if len(recs) == 0 {
+ return "", fmt.Errorf("%s has no instances, so there is no database to read", app)
+ }
+ hosts := make([]string, 0, len(recs))
+ for _, r := range recs {
+ hosts = append(hosts, r.Host)
+ }
+ sort.Strings(hosts)
+ more := ""
+ if truncated {
+ more = ", and more"
+ }
+ return "", fmt.Errorf("%s has %d instances. Name one with --instance: %s%s", app, len(recs), strings.Join(hosts, ", "), more)
+}
+
+// dbRestore restores the target's replica into a private directory under
+// the system temp dir. The caller removes Restored.Dir.
+func dbRestore(ctx context.Context, t dbTarget) (*dbdoor.Restored, error) {
+ r, err := dbdoor.RestoreTo(ctx, t.sess.dbClient(t.acct, t.app, t.host), "")
+ if err != nil {
+ return nil, dbRestoreError(t.host, err)
+ }
+ return r, nil
+}
+
+// dbRestoreError turns a failed restore into the sentence a member needs.
+// A refusal from the door carries the console's own words; the others are
+// named here so none of them reads as a transient "no replica found".
+func dbRestoreError(host string, err error) error {
+ var door *dbdoor.DoorError
+ switch {
+ case errors.As(err, &door) && door.Status == 404:
+ return fmt.Errorf("no instance %s that you can read", host)
+ case errors.As(err, &door):
+ return errors.New(door.Error())
+ case errors.Is(err, dbdoor.ErrNothingReplicated):
+ return fmt.Errorf("the platform has no copy of %s's database yet. Nothing has been replicated. "+
+ "If the app replicates its own database (replicate: \"app\"), the platform never will", host)
+ case errors.Is(err, dbdoor.ErrCheckFailed):
+ return fmt.Errorf("the copy of %s's database failed its integrity check, so it was not kept: %v", host, err)
+ case errors.Is(err, os.ErrNotExist):
+ return fmt.Errorf("the replica of %s changed while it was being read, twice. Try again", host)
+ case errors.Is(err, context.Canceled):
+ return errors.New("stopped")
+ }
+ return fmt.Errorf("could not restore %s's database: %w", host, err)
+}
+
+// dbAsOf renders the copy's newest transaction and its time (R1).
+func dbAsOf(r *dbdoor.Restored) string {
+ if r.At.IsZero() {
+ return fmt.Sprintf("transaction %d", r.TXID)
+ }
+ return fmt.Sprintf("transaction %d, written %s", r.TXID, r.At.UTC().Format("2006-01-02 15:04:05 UTC"))
+}
+
+func runDBPull(args []string) {
+ fs := flag.NewFlagSet("db pull", flag.ExitOnError)
+ target := dbFlags(fs, "pull")
+ out := fs.String("out", "", "where to save the copy; must not exist yet")
+ fs.Usage = flagUsage(fs, dbUsage)
+ fs.Parse(args)
+ if strings.TrimSpace(*out) == "" {
+ fmt.Fprint(os.Stderr, "carlos db pull: --out is required: name a path for the copy\n\n"+dbUsage)
+ os.Exit(2)
+ }
+ if _, err := os.Lstat(*out); err == nil {
+ fmt.Fprintf(os.Stderr, "carlos db pull: %s already exists. Choose a new path for --out\n", *out)
+ os.Exit(2)
+ }
+ t := target()
+ ctx, stop := dbSignalContext()
+ defer stop()
+ if err := dbPull(ctx, t, *out); err != nil {
+ stop()
+ log.Fatalf("carlos db pull: %v", err)
+ }
+}
+
+// dbPull restores, copies to out, and removes the private directory on
+// every path. It returns rather than exits so that cleanup always runs.
+func dbPull(ctx context.Context, t dbTarget, out string) error {
+ r, err := dbRestore(ctx, t)
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(r.Dir)
+ if err := copyOut(ctx, r.Path, out); err != nil {
+ return err
+ }
+ fmt.Fprintf(stdout, "Saved a copy of %s's database to %s.\n", t.host, out)
+ fmt.Fprintf(stdout, "It holds every change up to %s.\n", dbAsOf(r))
+ fmt.Fprintf(stdout, "It is a copy. Nothing you do to it reaches %s.\n", t.host)
+ return nil
+}
+
+// copyOut copies the restored database to out, 0600, refusing an existing
+// out. A copy and never os.Rename: the temp directory and the user's
+// destination are routinely on different filesystems, where a rename
+// fails. A failed copy leaves nothing behind at out.
+func copyOut(ctx context.Context, src, out string) (err error) {
+ // Nothing is published once the person has asked to stop.
+ if err := ctx.Err(); err != nil {
+ return err
+ }
+ in, err := os.Open(src)
+ if err != nil {
+ return err
+ }
+ defer in.Close()
+ f, err := os.OpenFile(out, os.O_WRONLY|os.O_CREATE|os.O_EXCL, 0o600)
+ if err != nil {
+ if errors.Is(err, os.ErrExist) {
+ return fmt.Errorf("%s already exists. Choose a new path for --out", out)
+ }
+ return err
+ }
+ defer func() {
+ if err != nil {
+ f.Close()
+ os.Remove(out)
+ }
+ }()
+ // The mode passed to OpenFile is filtered by the umask; 0600 is what
+ // the spec promises, so it is set, not hoped for.
+ if err = f.Chmod(0o600); err != nil {
+ return err
+ }
+ if _, err = io.Copy(f, ctxReader{ctx, in}); err != nil {
+ return err
+ }
+ if err = f.Sync(); err != nil {
+ return err
+ }
+ if err = f.Close(); err != nil {
+ return err
+ }
+ // A stop that lands as the copy finishes still withdraws it: the
+ // caller reports success only for a copy that was not interrupted.
+ if err = ctx.Err(); err != nil {
+ os.Remove(out)
+ return err
+ }
+ return nil
+}
+
+// ctxReader stops a copy at the next read once ctx is done.
+type ctxReader struct {
+ ctx context.Context
+ r io.Reader
+}
+
+func (c ctxReader) Read(p []byte) (int, error) {
+ if err := c.ctx.Err(); err != nil {
+ return 0, err
+ }
+ return c.r.Read(p)
+}
+
+func runDBQuery(args []string) {
+ fs := flag.NewFlagSet("db query", flag.ExitOnError)
+ target := dbFlags(fs, "query")
+ sqlText := fs.String("sql", "", "the query to run")
+ fs.Usage = flagUsage(fs, dbUsage)
+ fs.Parse(args)
+ if strings.TrimSpace(*sqlText) == "" {
+ fmt.Fprint(os.Stderr, "carlos db query: --sql is required\n\n"+dbUsage)
+ os.Exit(2)
+ }
+ t := target()
+ ctx, stop := dbSignalContext()
+ defer stop()
+ if err := dbQuery(ctx, t, *sqlText); err != nil {
+ stop()
+ log.Fatalf("carlos db query: %v", err)
+ }
+}
+
+func dbQuery(ctx context.Context, t dbTarget, sqlText string) error {
+ r, err := dbRestore(ctx, t)
+ if err != nil {
+ return err
+ }
+ defer os.RemoveAll(r.Dir)
+ fmt.Fprintf(os.Stderr, "Querying a copy of %s's database, up to %s. Nothing you run here reaches %s.\n",
+ t.host, dbAsOf(r), t.host)
+ return runQueryContext(ctx, stdout, r.Path, sqlText)
+}
+
+func runQuery(w io.Writer, path, sqlText string) error {
+ return runQueryContext(context.Background(), w, path, sqlText)
+}
+
+// runQueryContext opens path read-only, both ways (mode=ro and
+// query_only, spec §4), runs sqlText, and writes a header row then one
+// tab-separated line per row. A write fails with SQLite's own "attempt to
+// write a readonly database", returned as it is: the error is the truth.
+func runQueryContext(ctx context.Context, w io.Writer, path, sqlText string) error {
+ if err := checkQuerySQL(sqlText); err != nil {
+ return err
+ }
+ // temp_store(2): sorts and temp b-trees stay in memory, so a large
+ // query cannot spill plaintext pages to a file outside the private
+ // directory.
+ db, err := sql.Open("sqlite", "file:"+path+"?mode=ro&_pragma=query_only(1)&_pragma=busy_timeout(5000)&_pragma=temp_store(2)")
+ if err != nil {
+ return err
+ }
+ defer db.Close()
+ conn, err := db.Conn(ctx)
+ if err != nil {
+ return err
+ }
+ defer conn.Close()
+ // mode=ro covers only main, and query_only is a pragma the statement
+ // list itself can switch off. ATTACH opens with the connection's
+ // read-write flags, so it is the way around both: a second name for the
+ // copy that writes silently, or a new file anywhere the user can write.
+ // VACUUM INTO attaches internally. No attached databases at all closes
+ // every one of those.
+ if _, err := sqlite.Limit(conn, sqlitelib.SQLITE_LIMIT_ATTACHED, 0); err != nil {
+ return err
+ }
+ rows, err := conn.QueryContext(ctx, sqlText)
+ if err != nil {
+ return err
+ }
+ defer rows.Close()
+ cols, err := rows.Columns()
+ if err != nil {
+ return err
+ }
+ if len(cols) > 0 {
+ if _, err := fmt.Fprintln(w, strings.Join(cols, "\t")); err != nil {
+ return err
+ }
+ }
+ vals := make([]any, len(cols))
+ ptrs := make([]any, len(cols))
+ for i := range vals {
+ ptrs[i] = &vals[i]
+ }
+ line := make([]string, len(cols))
+ for rows.Next() {
+ if err := rows.Scan(ptrs...); err != nil {
+ return err
+ }
+ for i, v := range vals {
+ line[i] = dbQueryCell(v)
+ }
+ // A reader that went away (EPIPE, once SIGPIPE is caught) ends the
+ // query here rather than after every remaining row.
+ if _, err := fmt.Fprintln(w, strings.Join(line, "\t")); err != nil {
+ return err
+ }
+ }
+ return rows.Err()
+}
+
+// dbQueryCell renders one value. NULL is spelled out so it cannot be mistaken
+// for an empty string; tabs and newlines inside a value are escaped so a
+// row stays one line.
+func dbQueryCell(v any) string {
+ var s string
+ switch x := v.(type) {
+ case nil:
+ return "NULL"
+ case []byte:
+ s = string(x)
+ case time.Time:
+ s = x.UTC().Format(time.RFC3339Nano)
+ default:
+ s = fmt.Sprint(x)
+ }
+ return strings.NewReplacer("\\", `\\`, "\t", `\t`, "\n", `\n`, "\r", `\r`).Replace(s)
+}
+
+// dbStopSignals end a db verb early. Each one cancels the context instead of
+// killing the process, so the deferred removal of the private directory
+// always runs: Ctrl-C, a closed terminal (SIGHUP), a supervisor's SIGTERM,
+// and SIGPIPE. The last is the one that matters most: `carlos db query ...
+// | head` closes stdout early, and with SIGPIPE left to the runtime Go
+// kills the process on the next write, which would leave a plaintext copy
+// of production in the temp directory. Caught, the write returns EPIPE
+// instead and the query loop stops on it.
+var dbStopSignals = []os.Signal{os.Interrupt, syscall.SIGTERM, syscall.SIGHUP, syscall.SIGPIPE}
+
+func dbSignalContext() (context.Context, context.CancelFunc) {
+ return signal.NotifyContext(context.Background(), dbStopSignals...)
+}
+
+// errOneStatement and errPragmaChange are query's two refusals before
+// SQLite sees anything.
+var (
+ errOneStatement = errors.New("carlos db query runs one statement at a time. Run each statement on its own")
+ errPragmaChange = errors.New("this pragma changes how the copy is read, so carlos db query does not run it. Reading a pragma's value is fine")
+)
+
+// dbReadPragmas are the pragmas that may take an argument here: each reads
+// the schema or checks the file, and none changes the connection. Any
+// other pragma may be read bare but not given a value.
+var dbReadPragmas = map[string]bool{
+ "table_info": true, "table_xinfo": true, "table_list": true,
+ "index_info": true, "index_xinfo": true, "index_list": true,
+ "foreign_key_list": true, "foreign_key_check": true,
+ "integrity_check": true, "quick_check": true,
+}
+
+// checkQuerySQL is the guard modernc.org/sqlite v1.55.0 leaves to its
+// caller: it exposes no authorizer and no sqlite3_stmt_readonly, and it runs
+// every statement in a list. mode=ro, query_only and the attach limit hold
+// for any ONE statement, so the rule is one statement, and no pragma that
+// changes the connection (query_only, temp_store, temp_store_directory,
+// writable_schema and the rest), since a pragma is the one statement that
+// can loosen what follows it.
+//
+// The pragma rule reads the statement's tokens, not its raw prefix: SQLite
+// skips byte-order marks, comments and whitespace anywhere, and applies a
+// pragma at prepare even under EXPLAIN. So when the statement is a PRAGMA
+// (first token, or after EXPLAIN [QUERY PLAN]), the pragma it names may be
+// given a value only if it is a read-only one, and only in the (argument)
+// form. Elsewhere "pragma" is an identifier and a read may use it.
+func checkQuerySQL(q string) error {
+ _, rest := splitFirstStatement(q)
+ if len(sqlTokens(rest)) > 0 {
+ return errOneStatement
+ }
+ toks := sqlTokens(q)
+ // A pragma is a statement, so it can only stand in statement position:
+ // first, or after EXPLAIN [QUERY PLAN]. Anywhere else "pragma" is an
+ // ordinary identifier (a column, an alias). One statement is enforced
+ // above, so there is no later statement position to check.
+ i := 0
+ if i < len(toks) && toks[i].kind == sqlWord && toks[i].text == "explain" {
+ i++
+ if i+1 < len(toks) && toks[i].kind == sqlWord && toks[i].text == "query" &&
+ toks[i+1].kind == sqlWord && toks[i+1].text == "plan" {
+ i += 2
+ }
+ }
+ if i < len(toks) && toks[i].kind == sqlWord && toks[i].text == "pragma" {
+ // PRAGMA [schema .] name [= value | (value)]
+ j := i + 1
+ name := ""
+ if j < len(toks) && toks[j].kind != sqlPunct {
+ name = toks[j].text
+ j++
+ }
+ if j+1 < len(toks) && toks[j].kind == sqlPunct && toks[j].text == "." && toks[j+1].kind != sqlPunct {
+ name = toks[j+1].text
+ j += 2
+ }
+ if j >= len(toks) || toks[j].kind != sqlPunct || (toks[j].text != "=" && toks[j].text != "(") {
+ return nil // a bare read of the pragma's value
+ }
+ if toks[j].text == "(" && dbReadPragmas[name] {
+ return nil
+ }
+ return errPragmaChange
+ }
+ return nil
+}
+
+type sqlTokenKind int
+
+const (
+ sqlWord sqlTokenKind = iota // keyword or bare identifier, lower case
+ sqlQuoted // string literal or quoted identifier, unquoted, lower case
+ sqlPunct // any other single byte
+)
+
+type sqlToken struct {
+ kind sqlTokenKind
+ text string
+}
+
+const utf8BOM = "\uFEFF"
+
+// sqlTokens splits q the way SQLite's tokenizer does, for the guard's
+// purposes: comments and whitespace (space, tab, newline, carriage return,
+// form feed, vertical tab, and a byte-order mark anywhere) separate tokens
+// and vanish; string literals and quoted identifiers ('...', "...", `...`,
+// [...]) are one token each; runs of identifier bytes are words; anything
+// else is one punctuation byte.
+func sqlTokens(q string) []sqlToken {
+ var out []sqlToken
+ for i := 0; i < len(q); {
+ c := q[i]
+ switch {
+ case strings.HasPrefix(q[i:], utf8BOM):
+ i += len(utf8BOM)
+ case c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f' || c == '\v':
+ i++
+ case c == '-' && i+1 < len(q) && q[i+1] == '-':
+ if j := strings.IndexByte(q[i:], '\n'); j >= 0 {
+ i += j + 1
+ } else {
+ i = len(q)
+ }
+ case c == '/' && i+1 < len(q) && q[i+1] == '*':
+ if j := strings.Index(q[i+2:], "*/"); j >= 0 {
+ i += j + 4
+ } else {
+ i = len(q)
+ }
+ case c == '\'' || c == '"' || c == '`':
+ end := min(skipQuoted(q, i, c), len(q))
+ body := strings.ReplaceAll(q[i+1:end], string([]byte{c, c}), string(c))
+ out = append(out, sqlToken{sqlQuoted, strings.ToLower(body)})
+ i = end + 1
+ case c == '[':
+ j := strings.IndexByte(q[i:], ']')
+ if j < 0 {
+ j = len(q) - i
+ }
+ out = append(out, sqlToken{sqlQuoted, strings.ToLower(q[i+1 : i+j])})
+ i += j + 1
+ case isSQLIdentByte(c):
+ j := i
+ for j < len(q) && isSQLIdentByte(q[j]) && !strings.HasPrefix(q[j:], utf8BOM) {
+ j++
+ }
+ out = append(out, sqlToken{sqlWord, strings.ToLower(q[i:j])})
+ i = j
+ default:
+ out = append(out, sqlToken{sqlPunct, string(c)})
+ i++
+ }
+ }
+ return out
+}
+
+// isSQLIdentByte reports a byte that can continue an unquoted identifier:
+// ASCII letters, digits, _ and $, and any byte of a multi-byte UTF-8
+// character (the byte-order mark is checked for before this).
+func isSQLIdentByte(c byte) bool {
+ return c == '_' || c == '$' || c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' || c >= 0x80
+}
+
+// splitFirstStatement returns the text up to and including the first
+// top-level semicolon, and everything after it. Semicolons inside string
+// literals, quoted identifiers and comments do not count.
+func splitFirstStatement(q string) (first, rest string) {
+ for i := 0; i < len(q); i++ {
+ switch c := q[i]; c {
+ case '\'', '"', '`':
+ i = skipQuoted(q, i, c)
+ case '[':
+ if j := strings.IndexByte(q[i:], ']'); j >= 0 {
+ i += j
+ } else {
+ i = len(q)
+ }
+ case '-':
+ if i+1 < len(q) && q[i+1] == '-' {
+ if j := strings.IndexByte(q[i:], '\n'); j >= 0 {
+ i += j
+ } else {
+ i = len(q)
+ }
+ }
+ case '/':
+ if i+1 < len(q) && q[i+1] == '*' {
+ if j := strings.Index(q[i+2:], "*/"); j >= 0 {
+ i += j + 3
+ } else {
+ i = len(q)
+ }
+ }
+ case ';':
+ return q[:i+1], q[i+1:]
+ }
+ }
+ return q, ""
+}
+
+// skipQuoted returns the index of the quote closing the one at i, where a
+// doubled quote is an escaped one, or len(q) when it never closes.
+func skipQuoted(q string, i int, quote byte) int {
+ for j := i + 1; j < len(q); j++ {
+ if q[j] == quote {
+ if j+1 < len(q) && q[j+1] == quote {
+ j++
+ continue
+ }
+ return j
+ }
+ }
+ return len(q)
+}
diff --git a/db_e2e_test.go b/db_e2e_test.go
new file mode 100644
index 000000000..29fad228b
--- /dev/null
+++ b/db_e2e_test.go
@@ -0,0 +1,249 @@
+// End-to-end acceptance for `carlos db` (spec 2026-09-02): a REAL
+// console.Server whose door reads a fake S3 holding a real litestream
+// replica, through the production replica client (lease.ReadReplicaClient
+// with CARLOS_S3_ENDPOINT), a device-flow login, then the real runDB verbs
+// driven in-process. The member's whole path, with no box access and no
+// AWS credentials of their own. TestE2ELogsThroughConsole's shape.
+package main
+
+import (
+ "bytes"
+ "context"
+ "encoding/json"
+ "encoding/xml"
+ "io/fs"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/superfly/ltx"
+
+ "github.com/carlosframework/platform/internal/console"
+ "github.com/carlosframework/platform/internal/dbdoor/dbdoortest"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+)
+
+// replicaS3 is just enough S3 for litestream's read path (ListObjectsV2 and
+// a ranged GetObject), serving file replicas under root in the S3 key
+// layout <prefix>/<level %04x>/<file>.ltx. It records every key asked for,
+// so a test can assert nothing outside the coordinates was ever requested.
+type replicaS3 struct {
+ t *testing.T
+ bucket string
+ root string
+ mu sync.Mutex
+ keys []string
+}
+
+func (s *replicaS3) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+ rest := strings.TrimPrefix(r.URL.Path, "/"+s.bucket)
+ if r.Method != http.MethodGet {
+ s.t.Errorf("fake S3: %s %s; the door must only read", r.Method, r.URL.Path)
+ w.WriteHeader(http.StatusForbidden)
+ return
+ }
+ if rest == "" || rest == "/" {
+ s.list(w, r.URL.Query().Get("prefix"))
+ return
+ }
+ key := strings.TrimPrefix(rest, "/")
+ s.mu.Lock()
+ s.keys = append(s.keys, key)
+ s.mu.Unlock()
+ f, err := os.Open(s.file(key))
+ if err != nil {
+ w.Header().Set("Content-Type", "application/xml")
+ w.WriteHeader(http.StatusNotFound)
+ w.Write([]byte(`<Error><Code>NoSuchKey</Code></Error>`))
+ return
+ }
+ defer f.Close()
+ http.ServeContent(w, r, "", time.Time{}, f)
+}
+
+// file maps <p>/<level hex>/<name> to the file client's <p>/ltx/<level>/<name>.
+func (s *replicaS3) file(key string) string {
+ dir, name := path.Split(key)
+ prefix, levelHex := path.Split(strings.TrimSuffix(dir, "/"))
+ level, err := strconv.ParseInt(levelHex, 16, 32)
+ if err != nil {
+ return filepath.Join(s.root, "nonexistent")
+ }
+ return filepath.Join(s.root, filepath.FromSlash(prefix), "ltx", strconv.Itoa(int(level)), name)
+}
+
+func (s *replicaS3) list(w http.ResponseWriter, prefix string) {
+ type obj struct {
+ Key string
+ Size int64
+ LastModified string
+ }
+ var objs []obj
+ dir, _ := path.Split(prefix)
+ p, levelHex := path.Split(strings.TrimSuffix(dir, "/"))
+ if level, err := strconv.ParseInt(levelHex, 16, 32); err == nil {
+ local := filepath.Join(s.root, filepath.FromSlash(p), "ltx", strconv.Itoa(int(level)))
+ ents, _ := os.ReadDir(local)
+ for _, e := range ents {
+ if _, _, err := ltx.ParseFilename(e.Name()); err != nil {
+ continue
+ }
+ fi, err := e.Info()
+ if err != nil {
+ continue
+ }
+ objs = append(objs, obj{Key: prefix + e.Name(), Size: fi.Size(), LastModified: fi.ModTime().UTC().Format(time.RFC3339)})
+ }
+ }
+ type result struct {
+ XMLName xml.Name `xml:"ListBucketResult"`
+ Name string
+ Prefix string
+ KeyCount int
+ IsTruncated bool
+ Contents []obj
+ }
+ w.Header().Set("Content-Type", "application/xml")
+ xml.NewEncoder(w).Encode(result{Name: s.bucket, Prefix: prefix, KeyCount: len(objs), Contents: objs})
+}
+
+func TestE2EDBThroughConsole(t *testing.T) {
+ useTempCredentials(t)
+ clearDeploymentEnv(t)
+ checkNoRegistry := requireNoRegistryTouch(t)
+
+ tmp := t.TempDir()
+ const host = "acme.go.tito.io"
+ replicaRoot := filepath.Join(tmp, "data")
+ dbdoortest.Replicate(t, filepath.Join(replicaRoot, "instances", host), 3)
+ // lock.json sits under the same prefix in production. Nothing may fetch it.
+ if err := os.WriteFile(filepath.Join(replicaRoot, "instances", host, "lock.json"), []byte(`{"owner":"box"}`), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ fake := &replicaS3{t: t, bucket: "data-test", root: replicaRoot}
+ s3srv := httptest.NewServer(fake)
+ t.Cleanup(s3srv.Close)
+ t.Setenv("CARLOS_S3_ENDPOINT", s3srv.URL)
+ t.Setenv("AWS_ACCESS_KEY_ID", "test-key")
+ t.Setenv("AWS_SECRET_ACCESS_KEY", "test-secret")
+ t.Setenv("AWS_EC2_METADATA_DISABLED", "true")
+
+ relStore := release.NewDirStore(filepath.Join(tmp, "store"))
+ seed, pubB64, err := release.GenerateKey()
+ if err != nil {
+ t.Fatal(err)
+ }
+ priv, err := release.ParseSigningKey(seed, "e2e")
+ if err != nil {
+ t.Fatal(err)
+ }
+ pub, err := release.ParsePublicKey(pubB64, "e2e")
+ if err != nil {
+ t.Fatal(err)
+ }
+ cst, err := console.Open(filepath.Join(tmp, "console.db"))
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(func() { cst.Close() })
+ var srv *console.Server
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ srv.ServeHTTP(w, r)
+ }))
+ t.Cleanup(ts.Close)
+ srv, err = console.NewServer(console.Config{
+ Origin: ts.URL,
+ Keymail: "http://keymail.invalid",
+ Registry: filepath.Join(tmp, "registry.db"),
+ Releases: relStore,
+ ReleasePub: pub,
+ Key: priv,
+ AppScheme: "http",
+ HibernateBucket: "data-test",
+ HibernateRegion: "us-east-1",
+ HibernatePrefix: "instances",
+ }, cst)
+ if err != nil {
+ t.Fatal(err)
+ }
+ const addr = "paul@keymail.dev"
+ if err := cst.AddMember(console.OpsAccountID, addr, console.RoleOwner, time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := cst.CreateAppIn(console.OpsAccountID, "titogo", addr, time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ yes := true
+ rec, _ := json.Marshal(provision.Record{Host: host, App: "titogo", Account: release.OpsAccount, Hibernate: &yes})
+ if _, err := relStore.Put(context.Background(), provision.RecordKey(release.OpsAccount, "titogo", host), rec, ""); err != nil {
+ t.Fatal(err)
+ }
+ token, expiresAt := e2eDeviceLogin(t, ts.URL, cst, addr, "e2e@laptop")
+ credFile, err := loadCredentials()
+ if err != nil {
+ t.Fatal(err)
+ }
+ credFile.put(credential{Console: ts.URL, Address: addr, Token: token, ExpiresAt: expiresAt})
+ if err := credFile.save(); err != nil {
+ t.Fatal(err)
+ }
+ t.Setenv("TMPDIR", t.TempDir())
+
+ // pull: the copy lands at --out, 0600, with every row, and the handover
+ // says it is a copy and names its newest transaction.
+ out := filepath.Join(t.TempDir(), "pulled.db")
+ buf := captureStdout(t)
+ runDB([]string{"pull", "--console", ts.URL, "--app", "titogo", "--account", "ops", "--instance", host, "--out", out})
+ said := buf.String()
+ for _, want := range []string{out, "copy", "transaction", host} {
+ if !strings.Contains(said, want) {
+ t.Errorf("pull said %q, want it to mention %q", said, want)
+ }
+ }
+ fi, err := os.Stat(out)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if fi.Mode().Perm() != 0o600 {
+ t.Errorf("pulled copy is %04o, want 0600", fi.Mode().Perm())
+ }
+ var rows bytes.Buffer
+ if err := runQuery(&rows, out, "SELECT count(*) AS n FROM t"); err != nil || rows.String() != "n\n"+strconv.Itoa(3*dbdoortest.Rows)+"\n" {
+ t.Fatalf("pulled copy = %q %v", rows.String(), err)
+ }
+
+ // query: rows on stdout, nothing else.
+ buf.Reset()
+ runDB([]string{"query", "--console", ts.URL, "--app", "titogo", "--account", "ops", "--instance", host, "--sql", "SELECT body FROM t ORDER BY id LIMIT 2"})
+ if got := buf.String(); got != "body\nrow-0-0\nrow-0-1\n" {
+ t.Errorf("query stdout = %q, want the header and two rows only", got)
+ }
+
+ // Nothing private is left in the temp dir, and nothing but LTX files
+ // under this instance's prefix was ever asked of the bucket.
+ _ = filepath.WalkDir(os.Getenv("TMPDIR"), func(p string, d fs.DirEntry, err error) error {
+ if err == nil && strings.HasPrefix(d.Name(), "carlos-db-") {
+ t.Errorf("restore directory left behind: %s", p)
+ }
+ return nil
+ })
+ fake.mu.Lock()
+ defer fake.mu.Unlock()
+ if len(fake.keys) == 0 {
+ t.Fatal("the door never read the bucket")
+ }
+ for _, k := range fake.keys {
+ if !strings.HasPrefix(k, "instances/"+host+"/") || !strings.HasSuffix(k, ".ltx") {
+ t.Errorf("the door asked the bucket for %q", k)
+ }
+ }
+ checkNoRegistry()
+}
diff --git a/db_test.go b/db_test.go
new file mode 100644
index 000000000..57c398e06
--- /dev/null
+++ b/db_test.go
@@ -0,0 +1,577 @@
+package main
+
+// The CLI half of `carlos db` (spec 2026-09-02 §4): the read-only query, the
+// pull's landing rules, and the wire shape the session's replica client
+// sends. The whole path through a real console is db_e2e_test.go.
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "database/sql"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "os/exec"
+ "path/filepath"
+ "runtime"
+ "slices"
+ "strings"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/dbdoor"
+)
+
+// fixtureDB writes a small database the way a restore leaves one: WAL mode,
+// one table, a few rows, checkpointed into the main file.
+func fixtureDB(t *testing.T) string {
+ t.Helper()
+ path := filepath.Join(t.TempDir(), "copy.db")
+ db, err := sql.Open("sqlite", path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := db.Exec(`PRAGMA journal_mode=WAL;
+CREATE TABLE users (id INTEGER PRIMARY KEY, name TEXT, note TEXT);
+INSERT INTO users (name, note) VALUES ('ada', NULL), ('grace', 'x');
+PRAGMA wal_checkpoint(TRUNCATE);`); err != nil {
+ t.Fatal(err)
+ }
+ if err := db.Close(); err != nil {
+ t.Fatal(err)
+ }
+ return path
+}
+
+func sha(t *testing.T, path string) [32]byte {
+ t.Helper()
+ b, err := os.ReadFile(path)
+ if err != nil {
+ t.Fatal(err)
+ }
+ return sha256.Sum256(b)
+}
+
+// A copy is harmless to write to. Read-only exists so a caller cannot
+// BELIEVE it changed production: an agent that sees "1 row affected"
+// concludes it fixed the bug. The error is the truth, in SQLite's words.
+func TestQueryRefusesAWriteWithSQLitesOwnError(t *testing.T) {
+ path := fixtureDB(t)
+ for _, stmt := range []string{
+ "UPDATE users SET name='x'",
+ "DELETE FROM users",
+ "INSERT INTO users (name) VALUES ('mallory')",
+ "CREATE TABLE t2 (x)",
+ } {
+ err := runQuery(&bytes.Buffer{}, path, stmt)
+ if err == nil || !strings.Contains(err.Error(), "attempt to write a readonly database") {
+ t.Errorf("%s: error = %v, want SQLite's readonly refusal", stmt, err)
+ }
+ }
+}
+
+// query_only is set as well as mode=ro, and a statement list cannot talk
+// its way past either: switching query_only off first still leaves the
+// write refused, and the data unchanged.
+func TestQueryCannotTurnOffItsOwnGuard(t *testing.T) {
+ path := fixtureDB(t)
+ for _, stmt := range []string{"PRAGMA query_only=0; UPDATE users SET name='x'", "PRAGMA query_only=0"} {
+ if err := runQuery(&bytes.Buffer{}, path, stmt); err == nil {
+ t.Fatalf("%q: succeeded, want a refusal", stmt)
+ }
+ }
+ var out bytes.Buffer
+ if err := runQuery(&out, path, "SELECT name FROM users ORDER BY id"); err != nil || out.String() != "name\nada\ngrace\n" {
+ t.Fatalf("read after = %v %q, want the rows unchanged", err, out.String())
+ }
+}
+
+func TestQueryLeavesTheFileByteIdentical(t *testing.T) {
+ path := fixtureDB(t)
+ before := sha(t, path)
+ _ = runQuery(&bytes.Buffer{}, path, "UPDATE users SET name='x'")
+ if err := runQuery(&bytes.Buffer{}, path, "SELECT * FROM users"); err != nil {
+ t.Fatal(err)
+ }
+ if sha(t, path) != before {
+ t.Fatal("the database file changed under a read-only query")
+ }
+}
+
+// Output is for an agent as much as a person: a header row, then one line
+// per row, tab-separated, NULL spelled out.
+func TestQueryPrintsAHeaderAndTabSeparatedRows(t *testing.T) {
+ path := fixtureDB(t)
+ var out bytes.Buffer
+ if err := runQuery(&out, path, "SELECT id, name, note FROM users ORDER BY id"); err != nil {
+ t.Fatal(err)
+ }
+ want := "id\tname\tnote\n1\tada\tNULL\n2\tgrace\tx\n"
+ if out.String() != want {
+ t.Fatalf("output = %q, want %q", out.String(), want)
+ }
+}
+
+func TestQueryReportsABadStatement(t *testing.T) {
+ if err := runQuery(&bytes.Buffer{}, fixtureDB(t), "SELEKT 1"); err == nil {
+ t.Fatal("a syntax error succeeded")
+ }
+}
+
+// --out must not exist, and what lands is 0600 whatever the umask.
+func TestPullRefusesAnExistingOutAndLands0600(t *testing.T) {
+ src := fixtureDB(t)
+ dir := t.TempDir()
+ existing := filepath.Join(dir, "taken.db")
+ if err := os.WriteFile(existing, []byte("mine"), 0o644); err != nil {
+ t.Fatal(err)
+ }
+ if err := copyOut(context.Background(), src, existing); err == nil {
+ t.Fatal("copyOut over an existing file succeeded")
+ }
+ if b, _ := os.ReadFile(existing); string(b) != "mine" {
+ t.Fatal("copyOut touched the existing file")
+ }
+ out := filepath.Join(dir, "new.db")
+ if err := copyOut(context.Background(), src, out); err != nil {
+ t.Fatal(err)
+ }
+ fi, err := os.Stat(out)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if perm := fi.Mode().Perm(); perm != 0o600 {
+ t.Fatalf("pulled copy is %04o, want 0600", perm)
+ }
+ if sha(t, out) != sha(t, src) {
+ t.Fatal("pulled copy differs from the restored one")
+ }
+}
+
+// os.Rename fails across devices, and the temp directory and --out are
+// routinely on different filesystems. A copy leaves its source in place; a
+// rename would not.
+func TestPullCopiesAcrossFilesystems(t *testing.T) {
+ src := fixtureDB(t)
+ out := filepath.Join(t.TempDir(), "elsewhere", "pulled.db")
+ if err := os.MkdirAll(filepath.Dir(out), 0o755); err != nil {
+ t.Fatal(err)
+ }
+ if err := copyOut(context.Background(), src, out); err != nil {
+ t.Fatal(err)
+ }
+ if _, err := os.Stat(src); err != nil {
+ t.Fatalf("the source is gone after copyOut (%v): that was a rename, which fails across filesystems", err)
+ }
+}
+
+// A failed copy leaves nothing at --out.
+func TestPullLeavesNoPartialFile(t *testing.T) {
+ out := filepath.Join(t.TempDir(), "pulled.db")
+ if err := copyOut(context.Background(), filepath.Join(t.TempDir(), "missing.db"), out); err == nil {
+ t.Fatal("copyOut of a missing source succeeded")
+ }
+ if _, err := os.Stat(out); !os.IsNotExist(err) {
+ t.Fatalf("a failed copy left %s behind (%v)", out, err)
+ }
+}
+
+// The session's replica client sends the coordinate under the app's db
+// path, carries the account segment and its ?exact=sqid switch verbatim,
+// and authenticates as the logged-in person.
+func TestAPISessionDBClientWireShape(t *testing.T) {
+ var gotURI, gotAuth, gotRange string
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ gotURI, gotAuth, gotRange = r.URL.RequestURI(), r.Header.Get("Authorization"), r.Header.Get("Range")
+ w.WriteHeader(http.StatusPartialContent)
+ }))
+ defer ts.Close()
+ c := newTestSession(ts.URL).dbClient(apiAcct{Seg: "bab", Exact: true}, "titogo", "a.go.tito.io")
+ rc, err := c.OpenLTXFile(context.Background(), 3, 16, 32, 0, 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ rc.Close()
+ if want := "/api/cli/apps/bab/titogo/db/a.go.tito.io" + dbdoor.GetTail(3, 16, 32) + "?exact=sqid"; gotURI != want {
+ t.Errorf("uri = %s, want %s", gotURI, want)
+ }
+ if gotAuth != "Bearer cli-token" || gotRange != "bytes=0-" {
+ t.Errorf("auth %q range %q", gotAuth, gotRange)
+ }
+}
+
+// Both guards are on, not one: mode=ro alone would pass every write test
+// above, so query_only is pinned on its own.
+func TestQueryRunsWithQueryOnlySet(t *testing.T) {
+ var out bytes.Buffer
+ if err := runQuery(&out, fixtureDB(t), "PRAGMA query_only"); err != nil || out.String() != "query_only\n1\n" {
+ t.Fatalf("PRAGMA query_only = %v %q, want 1", err, out.String())
+ }
+}
+
+// --instance may be left out only when the app has exactly one. Several is
+// a refusal that names them all; guessing would hand over the wrong data
+// while saying it was the right data (spec §7).
+func TestResolveDBInstance(t *testing.T) {
+ answer := ""
+ ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/api/cli/apps/bab/titogo/instances" {
+ t.Errorf("path = %s", r.URL.Path)
+ }
+ w.Write([]byte(answer))
+ }))
+ defer ts.Close()
+ sess := newTestSession(ts.URL)
+ ctx := context.Background()
+ acct := apiAcct{Seg: "bab"}
+
+ if h, err := resolveDBInstance(ctx, sess, acct, "titogo", "named.go.tito.io"); err != nil || h != "named.go.tito.io" {
+ t.Fatalf("named = %q %v", h, err)
+ }
+ answer = `{"instances":[{"host":"only.go.tito.io"}]}`
+ if h, err := resolveDBInstance(ctx, sess, acct, "titogo", ""); err != nil || h != "only.go.tito.io" {
+ t.Fatalf("one instance = %q %v", h, err)
+ }
+ answer = `{"instances":[{"host":"b.go.tito.io"},{"host":"a.go.tito.io"}]}`
+ _, err := resolveDBInstance(ctx, sess, acct, "titogo", "")
+ if err == nil || !strings.Contains(err.Error(), "--instance") || !strings.Contains(err.Error(), "a.go.tito.io, b.go.tito.io") {
+ t.Fatalf("two instances = %v, want a refusal naming both", err)
+ }
+ answer = `{"instances":[{"host":"a.go.tito.io"}],"truncated":true}`
+ if _, err := resolveDBInstance(ctx, sess, acct, "titogo", ""); err == nil {
+ t.Fatal("a truncated list of one was taken as the only instance")
+ }
+ answer = `{"instances":[]}`
+ if _, err := resolveDBInstance(ctx, sess, acct, "titogo", ""); err == nil {
+ t.Fatal("no instances resolved to something")
+ }
+}
+
+// Every way a restore can fail is named, and none of them reads as a
+// transient "no replica found" (spec §5 R4, §7).
+func TestDBRestoreErrorsNameTheirCause(t *testing.T) {
+ cases := []struct {
+ err error
+ want string
+ }{
+ {&dbdoor.DoorError{Status: 409, Message: "a.go.tito.io keeps its data in region uk."}, "region uk"},
+ {&dbdoor.DoorError{Status: 404}, "no instance"},
+ // A proxy's HTML page during a console roll carries no sentence;
+ // the status still has to reach the person.
+ {&dbdoor.DoorError{Status: 502}, "502"},
+ {dbdoor.ErrNothingReplicated, `replicate: "app"`},
+ {dbdoor.ErrCheckFailed, "integrity check"},
+ {os.ErrNotExist, "changed while it was being read"},
+ }
+ for _, c := range cases {
+ if got := dbRestoreError("a.go.tito.io", c.err).Error(); !strings.Contains(got, c.want) {
+ t.Errorf("%v -> %q, want it to mention %q", c.err, got, c.want)
+ }
+ }
+}
+
+// mode=ro covers only main, and query_only is a pragma a statement list can
+// switch off. ATTACH opens with the connection's read-write flags, so it is
+// the door around both: it could write the copy silently through a second
+// name, or create and fill a file anywhere the user can write. VACUUM INTO
+// attaches internally. All of them must fail, and leave no file behind.
+func TestQueryCannotAttachOrWriteFilesElsewhere(t *testing.T) {
+ path := fixtureDB(t)
+ elsewhere := t.TempDir()
+ stray := filepath.Join(elsewhere, "stray.db")
+ for _, stmt := range []string{
+ "PRAGMA query_only=0; ATTACH (SELECT file FROM pragma_database_list WHERE name='main') AS rw; UPDATE rw.users SET name='x'",
+ "ATTACH '" + stray + "' AS e",
+ "PRAGMA query_only=0; ATTACH '" + stray + "' AS e; CREATE TABLE e.t (x); INSERT INTO e.t VALUES (1)",
+ "VACUUM INTO '" + stray + "'",
+ "PRAGMA query_only=0; VACUUM INTO '" + stray + "'",
+ } {
+ if err := runQuery(&bytes.Buffer{}, path, stmt); err == nil {
+ t.Errorf("%s: succeeded, want a refusal", stmt)
+ }
+ if _, err := os.Stat(stray); !os.IsNotExist(err) {
+ t.Fatalf("%s: left a file at %s", stmt, stray)
+ }
+ }
+ var out bytes.Buffer
+ if err := runQuery(&out, path, "SELECT name FROM users ORDER BY id"); err != nil || out.String() != "name\nada\ngrace\n" {
+ t.Fatalf("rows after = %v %q, want them unchanged", err, out.String())
+ }
+}
+
+// errWriter fails every write, the way a closed pipe does.
+type errWriter struct{ n int }
+
+func (w *errWriter) Write([]byte) (int, error) { w.n++; return 0, errors.New("broken pipe") }
+
+// A reader that went away is an error the query loop stops on, not one it
+// prints a thousand more rows into.
+func TestQueryStopsWhenTheReaderGoesAway(t *testing.T) {
+ w := &errWriter{}
+ if err := runQuery(w, fixtureDB(t), "SELECT * FROM users"); err == nil {
+ t.Fatal("a failing writer was not reported")
+ }
+ if w.n != 1 {
+ t.Fatalf("kept writing after the first failure: %d writes", w.n)
+ }
+}
+
+// `carlos db query ... | head` closes stdout early. Without SIGPIPE caught,
+// Go kills the process on the next write and every deferred cleanup is
+// skipped, leaving a plaintext copy of production in the temp directory.
+// The helper below runs dbSignalContext's handling in a child whose stdout
+// is a pipe nobody reads; it must get EPIPE back, clean up, and exit 0.
+func TestQueryCleansUpWhenStdoutIsClosed(t *testing.T) {
+ if os.Getenv("CARLOS_DB_PIPE_HELPER") != "" {
+ ctx, stop := dbSignalContext()
+ defer stop()
+ dir := os.Getenv("CARLOS_DB_PIPE_HELPER")
+ for i := 0; i < 1<<20 && ctx.Err() == nil; i++ {
+ if _, err := os.Stdout.Write([]byte("row\n")); err != nil {
+ break
+ }
+ }
+ os.RemoveAll(dir)
+ os.Exit(0)
+ }
+ if runtime.GOOS == "windows" {
+ t.Skip("no SIGPIPE on windows")
+ }
+ dir := t.TempDir()
+ private := filepath.Join(dir, "carlos-db-x")
+ if err := os.Mkdir(private, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ r, w, err := os.Pipe()
+ if err != nil {
+ t.Fatal(err)
+ }
+ r.Close() // the reader has gone, like head after its lines
+ cmd := exec.Command(os.Args[0], "-test.run=^TestQueryCleansUpWhenStdoutIsClosed$")
+ cmd.Env = append(os.Environ(), "CARLOS_DB_PIPE_HELPER="+private)
+ cmd.Stdout = w
+ err = cmd.Run()
+ w.Close()
+ if err != nil {
+ t.Fatalf("helper died (%v): the write killed it before cleanup could run", err)
+ }
+ if _, err := os.Stat(private); !os.IsNotExist(err) {
+ t.Fatalf("private directory survived a closed stdout (%v)", err)
+ }
+}
+
+// dirEntries lists a directory's names, sorted.
+func dirEntries(t *testing.T, dir string) []string {
+ t.Helper()
+ ents, err := os.ReadDir(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var out []string
+ for _, e := range ents {
+ out = append(out, e.Name())
+ }
+ return out
+}
+
+// The review's reproduction, verbatim. mode=ro covers main only, the
+// attach limit does not cover the built-in temp schema, and the driver runs
+// every statement in a list, so switching query_only off first let a temp
+// copy of the data be written and changed with no error. It must fail, and
+// leave no file anywhere: not beside the copy, not in the temp directory.
+func TestQueryRefusesTheTempTableBypass(t *testing.T) {
+ tmp := t.TempDir()
+ t.Setenv("TMPDIR", tmp)
+ path := fixtureDB(t)
+ before := dirEntries(t, filepath.Dir(path))
+ err := runQuery(&bytes.Buffer{}, path,
+ "PRAGMA query_only=0; CREATE TEMP TABLE stolen AS SELECT * FROM main.users; UPDATE stolen SET name='x'")
+ if err == nil {
+ t.Fatal("the temp-table bypass succeeded")
+ }
+ if got := dirEntries(t, tmp); len(got) != 0 {
+ t.Fatalf("temp directory holds %v after the refusal", got)
+ }
+ after := dirEntries(t, filepath.Dir(path))
+ for _, name := range after {
+ if !slices.Contains(before, name) && name != "copy.db-wal" && name != "copy.db-shm" {
+ t.Fatalf("a new file %s appeared beside the copy", name)
+ }
+ }
+}
+
+// One statement per query. The guards that matter (mode=ro, query_only)
+// hold for a single statement; a list is how one statement turns them off
+// for the next.
+func TestQueryRunsOneStatementOnly(t *testing.T) {
+ path := fixtureDB(t)
+ for _, stmt := range []string{
+ "SELECT 1; SELECT 2",
+ "PRAGMA query_only=0; UPDATE users SET name='x'",
+ "SELECT 1;\n-- trailing comment\nDELETE FROM users",
+ "SELECT 1; /* c */ SELECT 2",
+ } {
+ if err := runQuery(&bytes.Buffer{}, path, stmt); err == nil || !strings.Contains(err.Error(), "one statement") {
+ t.Errorf("%q: error = %v, want the one-statement refusal", stmt, err)
+ }
+ }
+ for _, stmt := range []string{
+ "SELECT name FROM users;",
+ "SELECT name FROM users; -- trailing comment",
+ "SELECT ';' AS semi, \"a;b\" FROM (SELECT 1 AS \"a;b\")",
+ "SELECT name FROM users /* ; */",
+ "WITH x AS (SELECT 1) SELECT * FROM x",
+ } {
+ if err := runQuery(&bytes.Buffer{}, path, stmt); err != nil {
+ t.Errorf("%q: error = %v, want it to run", stmt, err)
+ }
+ }
+}
+
+// A pragma that takes an argument can change how this connection reads or
+// where it spills (query_only, temp_store, temp_store_directory,
+// writable_schema...). Only the read-only introspection pragmas may take
+// one; bare reads of any pragma are fine.
+func TestQueryRefusesPragmaChanges(t *testing.T) {
+ path := fixtureDB(t)
+ for _, stmt := range []string{
+ "PRAGMA query_only=0",
+ "PRAGMA query_only = OFF",
+ "pragma temp_store=FILE",
+ "PRAGMA temp_store_directory='/tmp/x'",
+ "PRAGMA main.writable_schema=1",
+ "PRAGMA journal_mode(DELETE)",
+ " /* lead */ PRAGMA user_version=9",
+ } {
+ if err := runQuery(&bytes.Buffer{}, path, stmt); err == nil || !strings.Contains(err.Error(), "pragma") {
+ t.Errorf("%q: error = %v, want the pragma refusal", stmt, err)
+ }
+ }
+ for _, stmt := range []string{
+ "PRAGMA table_info(users)",
+ "PRAGMA main.table_info('users')",
+ "PRAGMA query_only",
+ "PRAGMA integrity_check",
+ "SELECT * FROM pragma_table_info('users')",
+ } {
+ if err := runQuery(&bytes.Buffer{}, path, stmt); err != nil {
+ t.Errorf("%q: error = %v, want it to run", stmt, err)
+ }
+ }
+}
+
+// Sorts and temp b-trees stay in memory, so a big query cannot spill
+// plaintext pages to a file outside the private directory.
+func TestQueryKeepsTempStorageInMemory(t *testing.T) {
+ var out bytes.Buffer
+ if err := runQuery(&out, fixtureDB(t), "PRAGMA temp_store"); err != nil || out.String() != "temp_store\n2\n" {
+ t.Fatalf("PRAGMA temp_store = %v %q, want 2 (memory)", err, out.String())
+ }
+}
+
+// SQLite applies a pragma when it prepares the statement, EXPLAIN or not,
+// and skips a leading byte-order mark and every kind of SQL whitespace. The
+// pragma check must see through all of them.
+func TestQueryRefusesDisguisedPragmaChanges(t *testing.T) {
+ path := fixtureDB(t)
+ for _, stmt := range []string{
+ "EXPLAIN PRAGMA query_only=0",
+ "explain query plan PRAGMA query_only=0",
+ "EXPLAIN\fQUERY\tPLAN\nPRAGMA temp_store=FILE",
+ "\uFEFFPRAGMA query_only=0",
+ "\uFEFF EXPLAIN /* x */ PRAGMA writable_schema=1",
+ "\fPRAGMA\fquery_only\f=\f0",
+ } {
+ if err := runQuery(&bytes.Buffer{}, path, stmt); err == nil || !strings.Contains(err.Error(), "pragma") {
+ t.Errorf("%q: error = %v, want the pragma refusal", stmt, err)
+ }
+ }
+ for _, stmt := range []string{"EXPLAIN SELECT * FROM users", "EXPLAIN QUERY PLAN SELECT * FROM users", "\uFEFFSELECT 1", "EXPLAIN PRAGMA table_info(users)"} {
+ if err := runQuery(&bytes.Buffer{}, path, stmt); err != nil {
+ t.Errorf("%q: error = %v, want it to run", stmt, err)
+ }
+ }
+}
+
+// A pull interrupted before or during the copy publishes nothing at --out,
+// and so cannot go on to report success.
+func TestPullPublishesNothingOnceCancelled(t *testing.T) {
+ src := fixtureDB(t)
+ out := filepath.Join(t.TempDir(), "pulled.db")
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ if err := copyOut(ctx, src, out); !errors.Is(err, context.Canceled) {
+ t.Fatalf("copyOut after cancel = %v, want context.Canceled", err)
+ }
+ if _, err := os.Stat(out); !os.IsNotExist(err) {
+ t.Fatalf("a cancelled pull left %s (%v)", out, err)
+ }
+}
+
+// The pragma guard is an invariant over the whole statement, not a prefix
+// check: SQLite skips byte-order marks, comments and every kind of
+// whitespace anywhere, and applies a pragma at prepare even under EXPLAIN.
+// This generates the disguises from parts and asserts every one is refused.
+func TestQueryRefusesEveryDisguiseOfAPragmaChange(t *testing.T) {
+ bom := "\uFEFF"
+ gaps := []string{"", " ", "\t", "\n", "\f", "\r\n", bom, " " + bom, bom + bom, "/*x*/", "/*x*/" + bom, bom + "/**/ ", "-- c\n", " /* a */ /* b */ "}
+ leads := []string{"", "EXPLAIN", "explain query plan", "EXPLAIN QUERY PLAN"}
+ changes := []struct{ name, set string }{
+ {"query_only", "=0"}, {"query_only", "(0)"}, {"temp_store", "=1"}, {"temp_store", "=FILE"},
+ {"temp_store_directory", "='/tmp/x'"}, {"writable_schema", "=1"}, {"main.writable_schema", "=ON"},
+ {`"query_only"`, "=0"}, {"[query_only]", "=0"}, {"`temp_store`", "=1"}, {"main . query_only", "=0"},
+ }
+ path := fixtureDB(t)
+ n := 0
+ for _, lead := range leads {
+ for _, g1 := range gaps {
+ for _, g2 := range gaps {
+ for _, c := range changes {
+ sep := g2
+ if sep == "" {
+ sep = " " // two words need something between them
+ }
+ stmt := g1 + lead + sep + "PRAGMA" + sep + c.name + g1 + c.set
+ if err := checkQuerySQL(stmt); err == nil || !strings.Contains(err.Error(), "pragma") {
+ t.Errorf("%q: error = %v, want the pragma refusal", stmt, err)
+ }
+ n++
+ }
+ }
+ }
+ }
+ if n < 1000 {
+ t.Fatalf("generated only %d cases", n)
+ }
+ // And end to end, a handful through the real query path.
+ for _, stmt := range []string{"/*x*/" + bom + "PRAGMA query_only=0", " " + bom + "PRAGMA query_only=0", bom + bom + "PRAGMA temp_store=1", "EXPLAIN " + bom + "PRAGMA query_only=0"} {
+ if err := runQuery(&bytes.Buffer{}, path, stmt); err == nil || !strings.Contains(err.Error(), "pragma") {
+ t.Errorf("%q through runQuery: error = %v, want the pragma refusal", stmt, err)
+ }
+ }
+ // What must still run.
+ for _, stmt := range []string{
+ "PRAGMA table_info(users)", bom + " PRAGMA table_info ( 'users' )", "PRAGMA query_only", "EXPLAIN PRAGMA query_only",
+ "SELECT 'PRAGMA query_only=0' AS s", `SELECT 1 AS "pragma"`, "SELECT 1 -- PRAGMA query_only=0", "SELECT 1 /* PRAGMA temp_store=1 */",
+ } {
+ if err := runQuery(&bytes.Buffer{}, path, stmt); err != nil {
+ t.Errorf("%q: error = %v, want it to run", stmt, err)
+ }
+ }
+}
+
+// "pragma" is only a statement in statement position. As a column name or
+// alias it is an ordinary identifier, and a read that uses it must run.
+func TestQueryAllowsPragmaAsAnIdentifier(t *testing.T) {
+ for _, q := range []string{
+ "SELECT pragma FROM (SELECT 1 AS pragma)",
+ "SELECT pragma = 1 FROM (SELECT 1 AS pragma)",
+ "SELECT x FROM (SELECT 1 AS x) WHERE x IN (SELECT 1 AS pragma)",
+ "EXPLAIN SELECT pragma = 1 FROM (SELECT 1 AS pragma)",
+ } {
+ if err := checkQuerySQL(q); err != nil {
+ t.Errorf("checkQuerySQL(%q) = %v, want it allowed", q, err)
+ }
+ }
+}
diff --git a/docs/site/cli.md b/docs/site/cli.md
index c173c7929..6a75143cc 100644
--- a/docs/site/cli.md
+++ b/docs/site/cli.md
@@ -397,6 +397,36 @@ that predates these flags does not understand a narrowed request: the console
answers that it has no such door, and a box that has not been rolled yet simply
does nothing. Neither one turns it into a restart of everything.
+### carlos db
+
+Reads a copy of an instance's database. The platform keeps a replica of
+every sleeping instance's database. This command restores that replica on
+your machine. It never touches the live database.
+
+```sh
+carlos db pull --app hello --out hello.db
+carlos db query --app hello --sql "SELECT count(*) FROM users"
+```
+
+`pull` saves the copy at the path you give with `--out`. The path must not
+exist yet, and the file is readable only by you. `query` restores a fresh
+copy, runs one query against it, and prints the rows to stdout with a header
+row, separated by tabs. The copy is deleted when the query finishes.
+
+Both say how recent the copy is: the last transaction it holds, and when
+that transaction was written. A sleeping instance's copy is as recent as its
+last sync before it went to sleep. A busy instance can be ahead of its copy.
+
+The copy is read-only. A query that tries to change it fails with SQLite's
+own error, `attempt to write a readonly database`. That is on purpose.
+Changing a copy would change nothing on the instance, and a command that
+seemed to succeed would say otherwise.
+
+If the app has more than one instance, name one with `--instance`. Only
+instances that sleep have a replica. An instance on your own fleet, in
+another region, or in a pool cannot be read this way, and the command says
+which.
+
### carlos schedule
Gives an app a timetable. A schedule is a time and a path: at each fire the
diff --git a/docs/superpowers/plans/2026-09-05-carlos-db-unit-1.md b/docs/superpowers/plans/2026-09-05-carlos-db-unit-1.md
new file mode 100644
index 000000000..4d91da366
--- /dev/null
+++ b/docs/superpowers/plans/2026-09-05-carlos-db-unit-1.md
@@ -0,0 +1,828 @@
+# `carlos db` Unit 1 Implementation Plan — the door, the client, `pull` and `query`
+
+> **For agentic workers:** REQUIRED SUB-SKILL: Use superpowers:subagent-driven-development (recommended) or superpowers:executing-plans to implement this plan task-by-task. Steps use checkbox (`- [ ]`) syntax for tracking.
+
+**Goal:** Let an account member pull a restored copy of their app's replicated SQLite to their own machine and query it read-only, with no AWS credentials and no box access.
+
+**Architecture:** The console gains an in-region, read-only S3 client on the data bucket and serves LTX files **by coordinate** (level + TXID range), never by key. The CLI implements litestream's `ReplicaClient` over that door with the three mutating methods stubbed to error, restores in-process into a 0700 temp directory, and opens the result read-only. The console never executes tenant SQL and never holds a restored database.
+
+**Tech Stack:** Go; litestream v0.5.14 as a library (`lease.Restore`, `lss3.ReplicaClient`); `modernc.org/sqlite` (pure Go, no cgo); the existing `internal/console` CLI-API gate.
+
+**Spec:** `docs/superpowers/specs/2026-09-02-carlos-db-design.md` — read it alongside this plan. §6 is the door, §12 is why the safety claim holds, §10 is the failure matrix every task draws its tests from.
+
+## Global Constraints
+
+- **Option A is decided** (spec §6): the console reads the region data bucket directly. Do **not** add an edges-file bucket lookup, and do **not** mirror through the deployment bucket.
+- **In-region, non-pooled, activator-replicated instances only.** Fleet-homed (`region == "fleet/<name>"`), cross-region, and `Pool != ""` are residency refusals made **before any S3 call**.
+- **`PrincipalUser` only.** Every service principal — legacy app token and every named service credential including `admin` — gets `http.NotFound`, never 403 (spec §6, "a 403 would be an oracle").
+- **The client must not implement `ReplicaClientV3`.** `Restore` type-asserts for it (`replica.go:612`) and would take the v0.3 path. A test asserts the negative.
+- **Never set `RestoreOptions.Follow`.** It writes a `-txid` sidecar and keeps polling.
+- **Run `lease.CheckDB` after every restore** rather than relying on `opt.IntegrityCheck`.
+- **The restore retry wraps the whole `Restore`**, never an individual read: `ResumableReader` treats `ErrNotExist` as fatal (`resumable_reader.go:72-75`), so a vanished planned file needs a fresh plan.
+- **`list` answers levels 0–9 inclusive**, returning an **empty list** (never 404) for levels the activator never writes. `SnapshotLevel = 9`.
+- **Listings ascend by `min_txid`.** `size`, `min_txid`, `max_txid` are mandatory on every entry.
+- **Store errors are logged, not returned** (`apischedule.go:145-147`) — they carry bucket names and request ids.
+- **No user-facing string ships without the `copy-review` gate.** Task 7 is where copy is finalised; earlier tasks may use placeholder wording in code but must not be merged to `main` with it.
+- **Every command needs a `docs/site/cli.md` heading** or `internal/docsite`'s `TestCLICommandsAreDocumented` fails.
+- Gate before commit: `go build ./...`, `go vet ./...`, `gofmt -l .` clean, `go test ./... -count=1`.
+
+---
+
+## File Structure
+
+**Create:**
+- `internal/dbdoor/coords.go` — the wire types shared by console and CLI (`LTXFile`, the list/get path builders). Leaf package, no console or CLI imports, so both sides agree on one shape.
+- `internal/dbdoor/coords_test.go`
+- `internal/dbdoor/client.go` — the CLI's read-only `litestream.ReplicaClient` over HTTP.
+- `internal/dbdoor/client_test.go`
+- `internal/dbdoor/restore.go` — restore into a 0700 temp dir, retry, `CheckDB`.
+- `internal/dbdoor/restore_test.go`
+- `internal/console/apidb.go` — resolution, refusals, and the two handlers.
+- `internal/console/apidb_test.go`
+- `db.go` — the `carlos db` CLI verb (`pull`, `query`).
+- `db_test.go`
+
+**Modify:**
+- `internal/lease/endpoint.go` — export a read-only replica-client constructor (currently `newReplicaClient`, unexported).
+- `internal/console/authcli.go` — two route registrations.
+- `cmd/console/main.go` — read `CARLOS_HIBERNATE_BUCKET`/`_REGION`/`_PREFIX` and `CARLOS_REGION`.
+- `internal/console/server.go` — carry the hibernate config and the S3 client on `Config`/`Server`.
+- `apimode.go` — session methods for the two door calls.
+- `main.go` — `case "db"` and the usage entry.
+- `docs/site/cli.md` — the `carlos db` heading.
+
+Why `internal/dbdoor` rather than putting the client in the root package: the console (server) and the CLI (client) must agree byte-for-byte on the wire shape, and a leaf package is the only way both can import it without `internal/console` and `main` importing each other. This mirrors `internal/dbverify`'s own reason for existing.
+
+---
+
+### Task 1: Export a read-only replica client from `internal/lease`
+
+The console needs `lss3.ReplicaClient` construction — endpoint override, path style, region — that `internal/lease` already gets right. Duplicating it is how the two drift.
+
+**Files:**
+- Modify: `internal/lease/endpoint.go:120-131`
+- Test: `internal/lease/endpoint_test.go`
+
+**Interfaces:**
+- Produces: `func ReadReplicaClient(bucket, prefix, region string) *lss3.ReplicaClient`
+
+- [ ] **Step 1: Write the failing test**
+
+```go
+// internal/lease/endpoint_test.go
+func TestReadReplicaClientCarriesBucketPrefixRegion(t *testing.T) {
+ c := ReadReplicaClient("data-bucket", "instances/a.example.com", "eu-west-1")
+ if c.Bucket != "data-bucket" {
+ t.Errorf("Bucket = %q, want data-bucket", c.Bucket)
+ }
+ if c.Path != "instances/a.example.com" {
+ t.Errorf("Path = %q, want instances/a.example.com", c.Path)
+ }
+ if c.Region != "eu-west-1" {
+ t.Errorf("Region = %q, want eu-west-1", c.Region)
+ }
+}
+```
+
+- [ ] **Step 2: Run test to verify it fails**
+
+Run: `go test ./internal/lease/ -run TestReadReplicaClient`
+Expected: FAIL — `undefined: ReadReplicaClient`
+
+- [ ] **Step 3: Write minimal implementation**
+
+```go
+// internal/lease/endpoint.go — beneath newReplicaClient
+
+// ReadReplicaClient is newReplicaClient for READERS outside this package —
+// the console's db door (spec 2026-09-02 §6). Identical construction on
+// purpose: a second hand-rolled client is how the endpoint override and the
+// path-style flag drift apart between the writer and the reader of the same
+// bucket.
+//
+// It returns the same type the writer uses, so nothing here can enforce
+// read-only by construction; that guarantee lives in internal/dbdoor's
+// client, which has no write methods at all.
+func ReadReplicaClient(bucket, prefix, region string) *lss3.ReplicaClient {
+ return newReplicaClient(bucket, prefix, region)
+}
+```
+
+- [ ] **Step 4: Run test to verify it passes**
+
+Run: `go test ./internal/lease/ -run TestReadReplicaClient`
+Expected: PASS
+
+- [ ] **Step 5: Commit**
+
+```bash
+git add internal/lease/endpoint.go internal/lease/endpoint_test.go
+git commit -m "lease: export ReadReplicaClient for the db door"
+```
+
+---
+
+### Task 2: The wire shape
+
+One package both sides import, so the door and the client cannot disagree about coordinates.
+
+**Files:**
+- Create: `internal/dbdoor/coords.go`
+- Test: `internal/dbdoor/coords_test.go`
+
+**Interfaces:**
+- Produces:
+ - `const MaxLevel = 9`
+ - `type LTXFile struct { Level int; MinTXID, MaxTXID uint64; Size int64; CreatedAt time.Time }`
+ - `type ListResponse struct { Files []LTXFile }`
+ - `func ListPath(acct, app, host string, level int) string`
+ - `func GetPath(acct, app, host string, level int, minTXID, maxTXID uint64) string`
+ - `func RangeHeader(offset, size int64) string`
+
+- [ ] **Step 1: Write the failing test**
+
+```go
+// internal/dbdoor/coords_test.go
+package dbdoor
+
+import "testing"
+
+// Range semantics must match litestream's S3 client exactly
+// (s3/replica_client.go:647-652), or a resumed read returns the wrong bytes.
+func TestRangeHeaderMatchesTheS3Client(t *testing.T) {
+ cases := []struct {
+ name string
+ offset, size int64
+ want string
+ }{
+ {"whole file from an offset", 0, 0, "bytes=0-"},
+ {"open-ended from an offset", 512, 0, "bytes=512-"},
+ {"sized read", 0, 100, "bytes=0-99"},
+ {"sized read at an offset", 512, 100, "bytes=512-611"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := RangeHeader(tc.offset, tc.size); got != tc.want {
+ t.Errorf("RangeHeader(%d, %d) = %q, want %q", tc.offset, tc.size, got, tc.want)
+ }
+ })
+ }
+}
+
+// Coordinates, never keys: the caller names a level and a TXID range, and
+// the server decides what object that is. lock.json has no coordinate.
+func TestGetPathIsCoordinateAddressed(t *testing.T) {
+ got := GetPath("bab", "titogo", "a.bab.oncarlos.com", 3, 16, 32)
+ for _, want := range []string{"bab", "titogo", "a.bab.oncarlos.com", "3", "16", "32"} {
+ if !strings.Contains(got, want) {
+ t.Fatalf("GetPath = %q, missing %q", got, want)
+ }
+ }
+ if strings.Contains(got, ".ltx") || strings.Contains(got, "lock") {
+ t.Fatalf("GetPath = %q leaks the key layout", got)
+ }
+}
+```
+
+- [ ] **Step 2: Run test to verify it fails**
+
+Run: `go test ./internal/dbdoor/ -run 'TestRangeHeader|TestGetPath'`
+Expected: FAIL — `undefined: RangeHeader`, `undefined: GetPath`
+
+- [ ] **Step 3: Write minimal implementation**
+
+```go
+// internal/dbdoor/coords.go
+
+// Package dbdoor is the wire shape of the `carlos db` replica door, shared
+// by the console that serves it and the CLI that reads it (spec
+// 2026-09-02 §6).
+//
+// It is a LEAF package for the reason internal/dbverify is one: the writer
+// (internal/console) and the reader (the CLI's main package) must agree on
+// one shape, and neither can import the other.
+//
+// Files are addressed by COORDINATE — level plus TXID range — never by key.
+// That is not decoration: lock.json lives under the same prefix as the LTX
+// files and is not a member's to fetch, and a coordinate has no way to name
+// it. See spec §6 "The door".
+package dbdoor
+
+import (
+ "fmt"
+ "net/url"
+ "time"
+)
+
+// MaxLevel is the highest level a list may name. litestream's
+// CalcRestorePlan asks for SnapshotLevel (9) first, then 8 down to 0
+// (replica.go:1452, 1476-1485). The activator writes only 0-3 and 9, so
+// most of that range is legitimately EMPTY — and empty is an empty list,
+// never a 404, or the restore stops at the first gap.
+const MaxLevel = 9
+
+// LTXFile is one replica file as the door reports it. Size is MANDATORY:
+// Restore rejects Size < ltx.HeaderSize (replica.go:630) and
+// ResumableReader uses it to detect a premature EOF
+// (resumable_reader.go:96).
+type LTXFile struct {
+ Level int `json:"level"`
+ MinTXID uint64 `json:"min_txid"`
+ MaxTXID uint64 `json:"max_txid"`
+ Size int64 `json:"size"`
+ CreatedAt time.Time `json:"created_at"`
+}
+
+// ListResponse is ordered by MinTXID ASCENDING — restoreLevelCursor assumes
+// filename order, which S3 gives because the names are zero-padded hex.
+type ListResponse struct {
+ Files []LTXFile `json:"files"`
+}
+
+func base(acct, app, host string) string {
+ return fmt.Sprintf("/api/cli/apps/%s/%s/db/%s",
+ url.PathEscape(acct), url.PathEscape(app), url.PathEscape(host))
+}
+
+func ListPath(acct, app, host string, level int) string {
+ return fmt.Sprintf("%s/ltx/%d", base(acct, app, host), level)
+}
+
+func GetPath(acct, app, host string, level int, minTXID, maxTXID uint64) string {
+ return fmt.Sprintf("%s/ltx/%d/%d-%d", base(acct, app, host), level, minTXID, maxTXID)
+}
+
+// RangeHeader mirrors litestream's own S3 client byte-for-byte
+// (s3/replica_client.go:647-652): open-ended when size is 0, inclusive
+// end otherwise.
+func RangeHeader(offset, size int64) string {
+ if size == 0 {
+ return fmt.Sprintf("bytes=%d-", offset)
+ }
+ return fmt.Sprintf("bytes=%d-%d", offset, offset+size-1)
+}
+```
+
+Add `"strings"` to the test's imports.
+
+- [ ] **Step 4: Run test to verify it passes**
+
+Run: `go test ./internal/dbdoor/`
+Expected: PASS
+
+- [ ] **Step 5: Commit**
+
+```bash
+git add internal/dbdoor/coords.go internal/dbdoor/coords_test.go
+git commit -m "dbdoor: the coordinate wire shape the door and client share"
+```
+
+---
+
+### Task 3: Resolution and the residency refusals
+
+The half of the door that makes **no S3 call**. Written and tested first because every refusal in the spec's matrix happens here, and none of them should ever reach a bucket.
+
+**Files:**
+- Create: `internal/console/apidb.go`
+- Test: `internal/console/apidb_test.go`
+
+**Interfaces:**
+- Consumes: `provision.Record` (fields `Host`, `App`, `Account`, `Pool`, `Region`), `provision.ValidHostShape`, `activator.HibernatePrefix`.
+- Produces: `func (s *Server) resolveReplica(rec provision.Record, host string) (prefix string, err error)` and `var ErrResidency, ErrPooled, ErrBadHost error`.
+
+- [ ] **Step 1: Write the failing test**
+
+```go
+// internal/console/apidb_test.go
+package console
+
+import (
+ "errors"
+ "testing"
+
+ "github.com/carlosframework/platform/internal/provision"
+)
+
+// Every one of these is refused BEFORE any S3 call. A door that reached the
+// bucket and mapped AccessDenied would leak both latency and the fact that
+// a bucket was tried.
+func TestResolveReplicaRefusesBeforeTouchingS3(t *testing.T) {
+ s := &Server{}
+ s.cfg.Region = "eu-west-1"
+ s.cfg.HibernatePrefix = "instances"
+
+ cases := []struct {
+ name string
+ rec provision.Record
+ host string
+ want error
+ }{
+ {
+ name: "another region",
+ rec: provision.Record{Host: "a.bab.oncarlos.com", Region: "us-east-1"},
+ host: "a.bab.oncarlos.com",
+ want: ErrResidency,
+ },
+ {
+ name: "fleet-homed",
+ rec: provision.Record{Host: "a.bab.oncarlos.com", Region: "fleet/acme"},
+ host: "a.bab.oncarlos.com",
+ want: ErrResidency,
+ },
+ {
+ name: "pooled",
+ rec: provision.Record{Host: "a.bab.oncarlos.com", Region: "eu-west-1", Pool: "shared"},
+ host: "a.bab.oncarlos.com",
+ want: ErrPooled,
+ },
+ {
+ name: "hostile host",
+ rec: provision.Record{Host: "../../etc", Region: "eu-west-1"},
+ host: "../../etc",
+ want: ErrBadHost,
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ _, err := s.resolveReplica(tc.rec, tc.host)
+ if !errors.Is(err, tc.want) {
+ t.Fatalf("resolveReplica = %v, want %v", err, tc.want)
+ }
+ })
+ }
+}
+
+// The happy path is an in-region, non-pooled instance, and the prefix is
+// the activator's own layout — not a string this package invents.
+func TestResolveReplicaBuildsTheActivatorPrefix(t *testing.T) {
+ s := &Server{}
+ s.cfg.Region = "eu-west-1"
+ s.cfg.HibernatePrefix = "instances"
+ got, err := s.resolveReplica(
+ provision.Record{Host: "a.bab.oncarlos.com", Region: "eu-west-1"},
+ "a.bab.oncarlos.com")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got != "instances/a.bab.oncarlos.com" {
+ t.Fatalf("prefix = %q, want instances/a.bab.oncarlos.com", got)
+ }
+}
+
+// A single-region deployment leaves both empty, and "" == "" must be a
+// match rather than a refusal — otherwise the door never works anywhere
+// that has not set CARLOS_REGION.
+func TestResolveReplicaTreatsEmptyRegionsAsAMatch(t *testing.T) {
+ s := &Server{}
+ s.cfg.HibernatePrefix = "instances"
+ if _, err := s.resolveReplica(
+ provision.Record{Host: "a.bab.oncarlos.com"}, "a.bab.oncarlos.com"); err != nil {
+ t.Fatalf("single-region deployment refused: %v", err)
+ }
+}
+```
+
+- [ ] **Step 2: Run test to verify it fails**
+
+Run: `go test ./internal/console/ -run TestResolveReplica`
+Expected: FAIL — `undefined: ErrResidency`, `s.resolveReplica undefined`, `s.cfg.Region undefined`
+
+- [ ] **Step 3: Write minimal implementation**
+
+Add to `internal/console/server.go`'s `Config` struct:
+
+```go
+ // The region data bucket the db door reads (spec 2026-09-02 §6,
+ // Option A). Read-only, in-region, instances/* only. Empty disables
+ // the door rather than failing it open.
+ HibernateBucket string
+ HibernateRegion string
+ HibernatePrefix string
+ // Region is this console's own placement, compared against an
+ // instance record's before any S3 call. Empty in a single-region
+ // deployment, where every record's is empty too.
+ Region string
+```
+
+Then `internal/console/apidb.go`:
+
+```go
+// The `carlos db` replica door (spec 2026-09-02 §6). This file is the half
+// that makes no S3 call: resolving an instance record to one replica
+// prefix, and refusing everything that is not this console's to read.
+//
+// Every refusal here happens BEFORE the bucket is touched. That is
+// deliberate — a door that tried and mapped AccessDenied would leak the
+// attempt, the latency, and eventually a bucket name into a log a member
+// can provoke.
+
+package console
+
+import (
+ "errors"
+ "fmt"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/provision"
+)
+
+var (
+ // ErrResidency: the data is in another region's bucket, or on a
+ // customer's own fleet box. The instance-move design makes those reads
+ // impossible at IAM by design, so this is the design working rather
+ // than a fault to retry.
+ ErrResidency = errors.New("db: the instance's data is outside this region")
+ // ErrPooled: a pooled record's data lives on the pool box, the same
+ // class movespass.go:150-153 refuses.
+ ErrPooled = errors.New("db: pooled instances have no replica of their own")
+ // ErrBadHost: path.Join cleans "../x" to "x" rather than refusing it,
+ // so a host becomes a key segment only after ValidHostShape.
+ ErrBadHost = errors.New("db: invalid host")
+)
+
+func (s *Server) resolveReplica(rec provision.Record, host string) (string, error) {
+ if !provision.ValidHostShape(host) {
+ return "", fmt.Errorf("%w: %q", ErrBadHost, host)
+ }
+ if rec.Pool != "" {
+ return "", fmt.Errorf("%w: %s is in pool %q", ErrPooled, host, rec.Pool)
+ }
+ if rec.Region != s.cfg.Region {
+ return "", fmt.Errorf("%w: %s is homed in %q, this console serves %q",
+ ErrResidency, host, rec.Region, s.cfg.Region)
+ }
+ return activator.HibernatePrefix(s.cfg.HibernatePrefix, host), nil
+}
+```
+
+Note: a `fleet/<name>` region is caught by the same `rec.Region != s.cfg.Region` compare, because a console's own region is never `fleet/…`. Keep the dedicated test case anyway — it documents the intent, and would catch a future change that special-cased the compare.
+
+- [ ] **Step 4: Run test to verify it passes**
+
+Run: `go test ./internal/console/ -run TestResolveReplica`
+Expected: PASS
+
+- [ ] **Step 5: Verify `ValidHostShape` is exported**
+
+Run: `grep -n "func ValidHostShape" internal/provision/provision.go`
+Expected: it exists and is exported. If it is not, export it in this task and note the rename in the commit.
+
+- [ ] **Step 6: Commit**
+
+```bash
+git add internal/console/apidb.go internal/console/apidb_test.go internal/console/server.go
+git commit -m "console: resolve an instance to one replica prefix, refusing before S3"
+```
+
+---
+
+### Task 4: The door handlers
+
+**Files:**
+- Modify: `internal/console/apidb.go`
+- Modify: `internal/console/authcli.go`
+- Test: `internal/console/apidb_test.go`
+
+**Interfaces:**
+- Consumes: Task 1's `lease.ReadReplicaClient`, Task 2's `dbdoor.LTXFile`/`ListResponse`/`MaxLevel`, Task 3's `resolveReplica`.
+- Produces: `POST`-free routes `GET /api/cli/apps/{account}/{app}/db/{host}/ltx/{level}` and `GET …/ltx/{level}/{min}-{max}`.
+
+- [ ] **Step 1: Write the failing test**
+
+```go
+// internal/console/apidb_test.go — append
+
+// A level the activator never writes is EMPTY, not missing. A 404 here
+// stops CalcRestorePlan at the first gap and the restore fails.
+func TestAPIDBListAnswersEmptyForUnwrittenLevels(t *testing.T) {
+ fx, token := dbFixture(t)
+ for level := 0; level <= dbdoor.MaxLevel; level++ {
+ status, out := get(t, fx.ts.URL+dbdoor.ListPath("bab", "titogo", dbTestHost, level), token)
+ if status != http.StatusOK {
+ t.Fatalf("level %d = %d %s, want 200 with an empty list", level, status, out)
+ }
+ }
+}
+
+// Coordinate addressing exists to make lock.json unreachable. This is the
+// test that keeps the door from decaying into an object proxy.
+func TestAPIDBServesNoKeyOutsideItsCoordinates(t *testing.T) {
+ fx, token := dbFixture(t)
+ for _, bad := range []string{
+ "/api/cli/apps/bab/titogo/db/" + dbTestHost + "/ltx/0/../../lock.json",
+ "/api/cli/apps/bab/titogo/db/" + dbTestHost + "/lock.json",
+ "/api/cli/apps/bab/titogo/db/" + dbTestHost + "/ltx/10",
+ } {
+ status, _ := get(t, fx.ts.URL+bad, token)
+ if status == http.StatusOK {
+ t.Fatalf("%s = 200, want a refusal", bad)
+ }
+ }
+}
+
+// Not a 403: a 403 tells any token holder that a db door exists for this
+// app. Service principals get the same answer as a stranger.
+func TestAPIDBRefusesServicePrincipalsWithA404(t *testing.T) {
+ fx, svcToken := dbServiceFixture(t)
+ status, _ := get(t, fx.ts.URL+dbdoor.ListPath("bab", "titogo", dbTestHost, 0), svcToken)
+ if status != http.StatusNotFound {
+ t.Fatalf("service principal = %d, want 404", status)
+ }
+}
+```
+
+Write `dbFixture`, `dbServiceFixture`, `dbTestHost` and `get` alongside, modelled on `scheduleFixture` and `postBody` in `apischedule_test.go`. The fixture must seed an `apps/<sqid>/<app>/instances/<host>.json` record with an empty region, and point `cfg.HibernateBucket` at a `release.NewDirStore`-backed fake or leave it empty so the empty-list path is exercised without S3.
+
+- [ ] **Step 2: Run test to verify it fails**
+
+Run: `go test ./internal/console/ -run TestAPIDB`
+Expected: FAIL — routes are 404 for every case, including the empty-list one that should be 200.
+
+- [ ] **Step 3: Write minimal implementation**
+
+Append to `internal/console/apidb.go`:
+
+```go
+// handleAPIDBList answers one level's listing. An out-of-range level is a
+// 404 (it is not a coordinate); a level with nothing in it is 200 and an
+// empty list (it is a coordinate that happens to be empty).
+func (s *Server) handleAPIDBList(w http.ResponseWriter, r *http.Request) {
+ prefix, acct, app, host, ok := s.dbGate(w, r)
+ if !ok {
+ return
+ }
+ level, err := strconv.Atoi(r.PathValue("level"))
+ if err != nil || level < 0 || level > dbdoor.MaxLevel {
+ http.NotFound(w, r)
+ return
+ }
+ client := lease.ReadReplicaClient(s.cfg.HibernateBucket, prefix, s.cfg.HibernateRegion)
+ itr, err := client.LTXFiles(r.Context(), level, 0, false)
+ if err != nil {
+ log.Printf("console: api db list %s/%s %s level %d: %v", acct, app, host, level, err)
+ jsonError(w, http.StatusBadGateway, "could not read the replica; try again")
+ return
+ }
+ defer itr.Close()
+ out := dbdoor.ListResponse{Files: []dbdoor.LTXFile{}}
+ for itr.Next() {
+ f := itr.Item()
+ out.Files = append(out.Files, dbdoor.LTXFile{
+ Level: level, MinTXID: uint64(f.MinTXID), MaxTXID: uint64(f.MaxTXID),
+ Size: f.Size, CreatedAt: f.CreatedAt,
+ })
+ }
+ sort.Slice(out.Files, func(i, j int) bool { return out.Files[i].MinTXID < out.Files[j].MinTXID })
+ writeJSON(w, http.StatusOK, out)
+}
+```
+
+and the `get` handler, which opens by coordinate and streams (F8: never a presigned redirect), honouring an inbound `Range` via `dbdoor.RangeHeader`'s inverse. `dbGate` wraps `apiScheduleApp(w, r, false, …)`, reads the instance record, calls `resolveReplica`, maps `ErrResidency`/`ErrPooled` to a 409 with the residency sentence and `ErrBadHost` to `http.NotFound`, and returns a 503 when `HibernateBucket` is empty.
+
+Register in `authcli.go` beside the schedule routes, but with `svcrole.Deny` — or, if no such helper exists, a `PrincipalUser`-only check inside `dbGate`; the test in Step 1 is the arbiter.
+
+- [ ] **Step 4: Run test to verify it passes**
+
+Run: `go test ./internal/console/ -run TestAPIDB`
+Expected: PASS
+
+- [ ] **Step 5: Commit**
+
+```bash
+git add internal/console/apidb.go internal/console/apidb_test.go internal/console/authcli.go
+git commit -m "console: serve replica LTX files by coordinate, members only"
+```
+
+---
+
+### Task 5: The read-only replica client
+
+The spec's central safety claim lives here (§6, §12). Two tests, not one: the mutating methods must **error**, and they must **never be called** during a real restore.
+
+**Files:**
+- Create: `internal/dbdoor/client.go`
+- Test: `internal/dbdoor/client_test.go`
+
+**Interfaces:**
+- Produces: `func NewClient(base, acct, app, host string, do func(*http.Request) (*http.Response, error)) *Client`, satisfying `litestream.ReplicaClient`.
+
+- [ ] **Step 1: Write the failing test**
+
+```go
+// internal/dbdoor/client_test.go
+package dbdoor
+
+import (
+ "context"
+ "testing"
+
+ "github.com/benbjohnson/litestream"
+)
+
+// The claim the whole design rests on: this client cannot corrupt a
+// replica, because it has no code that writes to one.
+func TestClientMutatingMethodsAlwaysError(t *testing.T) {
+ c := NewClient("http://console.invalid", "bab", "titogo", "a.example.com", nil)
+ ctx := context.Background()
+ if _, err := c.WriteLTXFile(ctx, 0, 1, 2, nil); err == nil {
+ t.Error("WriteLTXFile returned nil — this client must never write")
+ }
+ if err := c.DeleteLTXFiles(ctx, nil); err == nil {
+ t.Error("DeleteLTXFiles returned nil — this client must never delete")
+ }
+ if err := c.DeleteAll(ctx); err == nil {
+ t.Error("DeleteAll returned nil — this client must never delete")
+ }
+}
+
+// Restore type-asserts for ReplicaClientV3 (replica.go:612) and would take
+// the v0.3 path if this satisfied it. It must not.
+func TestClientIsNotAV3ReplicaClient(t *testing.T) {
+ var c any = NewClient("http://console.invalid", "bab", "titogo", "a.example.com", nil)
+ if _, ok := c.(litestream.ReplicaClientV3); ok {
+ t.Fatal("client satisfies ReplicaClientV3 — Restore would take the v0.3 path")
+ }
+}
+
+var _ litestream.ReplicaClient = (*Client)(nil)
+```
+
+- [ ] **Step 2: Run test to verify it fails**
+
+Run: `go test ./internal/dbdoor/ -run TestClient`
+Expected: FAIL — `undefined: NewClient`
+
+- [ ] **Step 3: Write minimal implementation**
+
+`Client` holds the base URL, the identifiers, and a `do` func (defaulting to `http.DefaultClient.Do`, injected in tests). Implement:
+
+- `Type() string` → `"carlos-db"`
+- `Init(ctx) error` → `nil` (idempotent no-op; the HTTP client needs no setup)
+- `SetLogger(*slog.Logger)` → store it
+- `LTXFiles(ctx, level, seek, useMetadata)` → GET `ListPath`, decode `ListResponse`, filter `>= seek` **locally** (seek is not on the wire), return an `ltx.FileIterator`
+- `OpenLTXFile(ctx, level, min, max, offset, size)` → GET `GetPath` with `Range: RangeHeader(offset, size)`; a 404 maps to `os.ErrNotExist`
+- `WriteLTXFile`, `DeleteLTXFiles`, `DeleteAll` → `return errReadOnly` unconditionally
+
+```go
+// errReadOnly is not a guard that can be switched off: these three methods
+// have no other body. The two-replicators hazard internal/caps and
+// internal/lease both warn about is unreachable from this client by
+// construction, and a future edit that wanted to reach it would have to add
+// the capability first.
+var errReadOnly = errors.New("dbdoor: this client is read-only — it restores replicas and never writes them")
+```
+
+- [ ] **Step 4: Run test to verify it passes**
+
+Run: `go test ./internal/dbdoor/ -run TestClient`
+Expected: PASS
+
+- [ ] **Step 5: Add the call-counting test — the other half**
+
+```go
+// The methods erroring is not enough: a restore that CALLED one would be
+// relying on an error path. This asserts a full restore never reaches them,
+// which is what catches a litestream upgrade changing the restore path.
+func TestRestoreNeverCallsAMutatingMethod(t *testing.T) {
+ // Serve a real replica from a DirStore-backed fake console (build one
+ // by replicating a small SQLite through lease.ReplicateWithLease into a
+ // temp dir, then serving its files over httptest).
+ // Wrap Client in a counter that increments per method name.
+ // Restore through it, then assert:
+ if counts["WriteLTXFile"]+counts["DeleteLTXFiles"]+counts["DeleteAll"] != 0 {
+ t.Fatalf("restore called a mutating method: %v", counts)
+ }
+}
+```
+
+- [ ] **Step 6: Run it, then commit**
+
+```bash
+go test ./internal/dbdoor/
+git add internal/dbdoor/client.go internal/dbdoor/client_test.go
+git commit -m "dbdoor: a replica client with no write methods"
+```
+
+---
+
+### Task 6: Restore into a private directory
+
+**Files:**
+- Create: `internal/dbdoor/restore.go`
+- Test: `internal/dbdoor/restore_test.go`
+
+**Interfaces:**
+- Produces: `func RestoreTo(ctx context.Context, c *Client, dir string) (path string, err error)`
+
+- [ ] **Step 1: Write the failing test**
+
+```go
+// litestream creates <out>.tmp itself with os.Create, at the process umask,
+// so a 0600 file cannot be pre-created for it. The containing directory is
+// the only thing that can be private — and it must also confine the -wal
+// and -shm a read-only open still needs.
+func TestRestoreToUsesAPrivateDirectory(t *testing.T) {
+ dir := t.TempDir()
+ // ... restore a known replica ...
+ fi, err := os.Stat(filepath.Dir(path))
+ if err != nil {
+ t.Fatal(err)
+ }
+ if perm := fi.Mode().Perm(); perm != 0o700 {
+ t.Fatalf("restore directory is %04o, want 0700", perm)
+ }
+}
+
+// A planned file can vanish under the restore when the writer's L0
+// retention fires. ResumableReader treats ErrNotExist as FATAL and does not
+// retry, so the retry has to wrap the whole Restore — a fresh plan is the
+// only recovery.
+func TestRestoreToRetriesTheWholeRestoreOnce(t *testing.T) {
+ // A client that returns os.ErrNotExist for the first OpenLTXFile and
+ // serves normally afterwards.
+ // Assert: RestoreTo succeeds, and the client saw two CalcRestorePlan
+ // listing passes rather than one.
+}
+```
+
+- [ ] **Step 2: Run to verify it fails**
+
+Run: `go test ./internal/dbdoor/ -run TestRestoreTo`
+Expected: FAIL — `undefined: RestoreTo`
+
+- [ ] **Step 3: Implement**
+
+`RestoreTo` creates `os.MkdirTemp(dir, "carlos-db-")`, chmods it 0700, builds `litestream.NewReplicaWithClient(nil, c)`, sets `RestoreOptions{OutputPath: …}` — **never `Follow`** — calls `Restore`, retries the whole call once on `os.ErrNotExist`, then runs `lease.CheckDB` on the result and returns the path.
+
+- [ ] **Step 4: Run to verify it passes, then commit**
+
+```bash
+go test ./internal/dbdoor/
+git add internal/dbdoor/restore.go internal/dbdoor/restore_test.go
+git commit -m "dbdoor: restore into a 0700 directory, retrying the whole plan once"
+```
+
+---
+
+### Task 7: `carlos db pull` and `carlos db query`
+
+**Files:**
+- Create: `db.go`, `db_test.go`
+- Modify: `apimode.go`, `main.go`, `docs/site/cli.md`
+
+- [ ] **Step 1: Write the failing tests**
+
+```go
+// A copy is harmless to write to. Read-only exists so a caller cannot
+// BELIEVE it changed production: an agent that sees "1 row affected"
+// concludes it fixed the bug. The error is the truth.
+func TestQueryRefusesAWriteWithSQLitesOwnError(t *testing.T) {
+ // open a restored fixture through the same helper db.go uses
+ err := runQuery(path, "UPDATE users SET name='x'")
+ if err == nil || !strings.Contains(err.Error(), "attempt to write a readonly database") {
+ t.Fatalf("write error = %v, want SQLite's readonly refusal", err)
+ }
+}
+
+func TestQueryLeavesTheFileByteIdentical(t *testing.T) { /* sha256 before and after */ }
+
+func TestPullRefusesAnExistingOutAndLands0600(t *testing.T) { /* both halves */ }
+
+func TestPullCopiesAcrossFilesystems(t *testing.T) {
+ // os.Rename fails across devices; the temp dir and --out routinely
+ // differ. Assert a copy path, not a rename.
+}
+```
+
+- [ ] **Step 2: Run to verify they fail, then implement**
+
+`db.go` adds `runDB(args)` dispatching `pull` and `query`; `apimode.go` gains a session method returning a configured `*dbdoor.Client`; `main.go` gains `case "db": runDB(args)` and a usage entry.
+
+- [ ] **Step 3: Add the `cli.md` heading**
+
+Required — `internal/docsite`'s `TestCLICommandsAreDocumented` fails without it. The prose is where "this is a copy, not your live database" gets said to the person most likely to assume otherwise.
+
+- [ ] **Step 4: Run the copy-review gate**
+
+**Do not merge without this.** Extract every user-facing string added in Tasks 3–7 — the residency refusal, the pooled refusal, the handover line, the read-only refusal, the `cli.md` prose — and run the `copy-review` skill. Apply the user's edits verbatim. Requirements R1–R6 in spec §5 are what the copy has to satisfy; R2 (age as a number carrying its bound, never an adjective, never defaulting to `stream`) lands in Unit 2 but the handover line's "this is a copy" half is Unit 1's.
+
+- [ ] **Step 5: Full gate, then commit**
+
+```bash
+go build ./... && go vet ./... && gofmt -l . && go test ./... -count=1
+git add db.go db_test.go apimode.go main.go docs/site/cli.md
+git commit -m "carlos db: pull and query a restored replica, read-only"
+```
+
+---
+
+## Self-Review
+
+**Spec coverage.** §6 configuration → Task 3 (`Config` fields) + Task 4. §6 resolution and refusals → Task 3. §6 door and wire specifics 1–5 → Tasks 2 and 4. §6 principal rule → Task 4. §6 read-only client → Task 5. §7 one-database → implicit: no `--db` flag exists anywhere in this plan. §12 conditions (i) no V3, (ii) no `Follow`, (iii) `CheckDB` → Task 5 Step 1 and Task 6 Step 3, and all three are in Global Constraints. §4 private directory and 0600 `--out` → Tasks 6 and 7. §4 read-only enforcement → Task 7.
+
+**Deliberately not in this plan** (Unit 2 and Unit 3 in spec §13): `ls` and every freshness field — `lease_held`, `sync_class` with its mandatory `unknown`, LTX-header age, the `dbverify` verdict, and the `--json` field set — plus `shell`. Wire specific 6 (`lease_held`) is Unit 2's, because nothing in `pull`/`query` reads it. Two follow-on plans, written once Unit 1 lands and its shapes are real rather than predicted.
+
+**Placeholders.** Tasks 4, 5 Step 5, 6 and 7 describe some test bodies rather than spelling them out — specifically the ones that need a real replica fixture. That fixture does not exist yet and its shape depends on Task 4's fixture helpers. **Before executing Task 4, write the fixture helper first and fill those bodies in.** Flagging rather than pretending: a test body invented here against a fixture that does not exist would be worse than a named gap.
+
+**Type consistency.** `dbdoor.LTXFile` fields are used identically in Tasks 2, 4 and 5. `lease.ReadReplicaClient(bucket, prefix, region)` is defined in Task 1 and called in Task 4 with that order. `resolveReplica(rec, host) (string, error)` is defined in Task 3 and consumed by `dbGate` in Task 4. `NewClient(base, acct, app, host, do)` is defined in Task 5 and consumed in Tasks 6 and 7.
diff --git a/docs/superpowers/specs/2026-09-02-carlos-db-design.md b/docs/superpowers/specs/2026-09-02-carlos-db-design.md
new file mode 100644
index 000000000..0cae85745
--- /dev/null
+++ b/docs/superpowers/specs/2026-09-02-carlos-db-design.md
@@ -0,0 +1,646 @@
+# `carlos db` — reading an app's database from its replica
+
+Design, 2026-09-02. Revised twice after review, same day (§14).
+
+## 1. What this is for
+
+There is no way to look at an app's data through the platform. The
+question "why does this row look wrong" currently has no `carlos` answer,
+which means it has an AWS answer — an SSM session onto the box and a
+`sqlite3` prompt against the live file. That violates the standing rule
+that every `carlos` command is designed for a user with no access to the
+underlying infra (AGENTS.md, Paul 2026-08-08), and it puts a hand-typed
+shell next to a production database for what is almost always a read.
+
+`carlos db` closes that gap by answering a deliberately narrower
+question. It does **not** connect you to the running database. It hands
+you a **restored copy of the replica** and lets you query that.
+
+The narrowing is the feature, not a compromise. It makes the command
+safe for a class of caller we specifically want to serve: an **agent**
+debugging a production problem. An agent that can read the data and
+cannot possibly change it is a useful colleague; an agent with a live
+`sqlite3` prompt on production is an incident waiting for a bad
+inference. Every design decision below is downstream of that.
+
+### Scope
+
+In: reading an app's platform-replicated SQLite, from the CLI, by an
+account member, as a local file, for **in-region, non-pooled,
+activator-replicated instances**.
+
+Out: writing to tenant data by any path; querying the live database;
+fleet-homed and cross-region instances (§13 — a v2 with a different
+transport); box-wide replicas; non-SQLite stores; the console web UI.
+
+## 2. The two decisions behind the shape
+
+**Replica, not live.** The alternative was a query door onto the running
+instance. That needs a synchronous console→instance path, which CARLOS
+does not have for ordinary boxes: the control plane is the bucket and
+boxes poll (`schedule.TickEvery` 15s, `logship` 10s, `control/serial`
+2s). Building one would mean a new transport *and* a new door onto the
+tenant process, to deliver a capability whose main use is a read. The
+replica is already there, already off-box, and already proven restorable
+by the restore drills.
+
+**Client-side query, not console-side.** The console serves replica
+**bytes**; the member's machine restores and queries. The console never
+executes tenant SQL, never holds a restored tenant database, and needs
+no query timeouts, row caps, or result-set memory budget.
+
+> The first draft justified this as "the console serves objects it
+> already stores". That is false — replicas live in customer data
+> buckets (§6). The client-side argument survives on its own merits, but
+> the transport is a genuinely new door, and §6 is the hardest part of
+> this design rather than the easiest.
+
+The client-side choice also buys the microcopy problem's best available
+answer (§5): the user is handed an actual file, so "this is a copy" is a
+true statement about an artifact in their hand rather than a warning
+against an illusion.
+
+Cost accepted: you transfer the whole database rather than one query's
+rows, and freshness is replica freshness rather than live.
+
+## 3. What already exists
+
+Accurate as of 2026-09-02. There are **two** replicators and they are
+easy to conflate:
+
+- **The activator's embedded library.** litestream v0.5.14 is a `go.mod`
+ dependency (not vendored — there is no `vendor/`), linked into the
+ `carlos` binary. It replicates **hibernating exec-backed instances**
+ under the lease. `lease.Restore` (`lease/lease.go:378`) restores from
+ it via litestream's nil-`*DB` URL-restore pattern (`lease.go:390`).
+ This is the set `carlos db` serves.
+- **The box-wide binary.** A separately pinned litestream **0.5.15**
+ executable (`ops/litestream.go:43`) driven by `/etc/litestream.yml`.
+ `carlos ops restore-verify` shells out to it.
+
+`ops.FleetDBs` is the box-wide unit's set and **excludes** hibernating
+exec routes (`ops/hibernate_verify.go:5-8`) — the complement of the
+activator's set, not the whole. Box-wide replica keys are absolute paths
+(`ops/litestream.go:170`) and **collide across boxes sharing a fleet
+prefix** (`/data/registry.db`), which is one reason they stay out of v1.
+
+Other parts this design uses:
+
+- `lease.VerifyReplica` restores without taking the lease;
+ `check.go:66-79` documents why that cannot shadow the true holder.
+ §12 records how far that reasoning actually reaches.
+- `activator.HibernatePrefix(prefix, host)` = `path.Join(prefix, host)`
+ (`activator.go:113`), prefix `instances` in production. Hosts are
+ unique FQDNs (`activator.go:118-121`), so prefixes cannot collide
+ across tenants.
+- `modernc.org/sqlite` v1.55.0 is a direct dependency and **pure Go**
+ (`go.mod:23`); litestream imports it, so the CLI already links it. The
+ query engine embeds — no `sqlite3` on the user's machine, no cgo.
+- `internal/dbverify` records restore-drill verdicts keyed by **host**
+ (`dbverify.go:45`), at one deployment-wide key (`dbverify.go:28`),
+ published only by the hibernation drill. `Merge` drops hosts outside
+ the writing box's fleet (`dbverify.go:60-80`), so the last box to run
+ overwrites the others. A missing verdict renders `unverified`.
+- Console gates: `instanceGate` (`apiinstances.go:70`), `s.audit`
+ (`server.go:2692`). The gate is **membership** via `AccountsFor(addr)`
+ (`apicli.go:208`) — people carry no owner/member role.
+
+**The console has no registry snapshot.** It opens the *local*
+`registry.db` on its own box (`console/view.go:532-538`). The
+`control/registry-snapshot/<box>.db` objects boxes write
+(`host/snapshot_loop.go:34`) are read by nothing in `internal/console`.
+Fleet-wide instance knowledge is `apps/<sqid>/<app>/instances/<host>.json`.
+§6 and §8 both depend on this distinction.
+
+## 4. Command surface
+
+```
+carlos db ls --app <a> [--account <a>]
+carlos db pull --app <a> [--instance <host>] --out <path>
+carlos db shell --app <a> [--instance <host>]
+carlos db query --app <a> [--instance <host>] --sql <text>
+```
+
+`ls` is the discovery verb and the one that makes the rest legible: per
+instance, the database the platform replicates, the age of the newest
+replicated write, whether the instance is replicating right now, its
+replication class *or* an honest "unknown" (§8), and its restore-drill
+verdict. It **also names databases declared in `data.dbs` that the
+platform does not replicate** (§7) — otherwise visible only in adopt
+logs a member cannot read.
+
+`pull` writes the restored copy to a path you name and stops. `--out` is
+**required** and must not already exist.
+
+`shell` restores into a private directory, opens an interactive prompt
+against it with the embedded engine, and removes the directory on exit.
+This is the closest thing to `rails console` a Go platform can honestly
+offer: a REPL over your *data*, which exists, rather than over your
+*app*, which cannot — Go is compiled ahead of time, and there is no
+interpreter in the running binary to evaluate against its live heap.
+
+`query` is the non-interactive form — one statement, results to stdout,
+exit code reflecting the query. This is the shape an agent uses and
+should pipe cleanly.
+
+### Where the copy lands
+
+litestream's `Restore` refuses an existing output path
+(`replica.go:604`) and creates `<out>.tmp` itself with `os.Create`, at
+the process umask (typically 0644). You **cannot** pre-create a 0600
+file for it to fill.
+
+So: always restore inside an `os.MkdirTemp` directory created 0700. That
+confines the `.tmp` and the `-wal`/`-shm` files a read-only open still
+needs (`check.go:96-98`). `shell` and `query` never leave that
+directory. `pull` restores there, then **copies** to `--out` with fsync
+and 0600 — a copy, not `os.Rename`, because the temp directory and the
+user's destination are routinely on different filesystems. Open
+question 2 is thereby answered: always 0600, and the user can loosen it.
+
+### Read-only is enforced, not assumed
+
+Open with `file:<path>?mode=ro` — the form `lease.CheckDB` already uses
+(`check.go:36`) — plus `PRAGMA query_only=1`.
+
+This matters most for the agent case, for a non-obvious reason. A copy
+is already harmless to write to, so enforcement is not about protecting
+production. It is about **not letting a caller believe it changed
+production**. An agent that runs an `UPDATE` against a writable copy
+sees `1 row affected` and concludes it fixed the bug. Against a
+read-only copy it gets SQLite's `attempt to write a readonly database`,
+and the error is the truth. Silent success on a discarded write is the
+single worst outcome this command could produce.
+
+§11 pins that error string in a test, so the claim stays true.
+
+## 5. The microcopy contract
+
+The safety property is a *fact the user has to hold in their head* —
+that this is a copy, of a stated age, and that nothing they do here
+reaches production. If they lose that fact, the command becomes a way to
+make confident wrong decisions. The copy is load-bearing.
+
+Exact strings are **not drafted here**. They go through the
+`copy-review` gate at implementation time, before being written into any
+source file. What follows is what each string must accomplish.
+
+**R1 — Say it at handover, and name the source exactly.** The line
+reporting a successful restore states that this is a copy, and names
+host, database, the newest replicated transaction id and its time. Two
+pulls must be distinguishable, and an agent must be able to cite an
+"as of".
+
+**R2 — Age is a number, and carries its expected bound — or admits it
+cannot.** Report the **age of the newest replicated write**, never a
+"lag" (which implies a live comparison the client cannot make, §8) and
+never an adjective. Beside it, report the replication class bound: an
+awake `daily` instance is legitimately up to 24h behind, exactly the
+case where an old copy is most easily mistaken for a current one.
+
+Where the class is **unknown** — which is the common case, because the
+console only knows it for routes on its own box (§8) — say so. It must
+never render as `stream` by default. A wrong bound is worse than no
+bound: it converts "I don't know" into "one second old".
+
+**R3 — The interactive prompt carries it too.** A `shell` session can
+outlive its banner by an hour. The prompt itself carries both the word
+copy/replica and the as-of time.
+
+**R4 — Refuse loudly, explain briefly.** Four refusals need their own
+voice: `replicate: "app"` (§7); the read-only write refusal (§4); the
+residency refusal (§6), which must name residency rather than read as a
+transient error; and the pooled-instance refusal (§6). Per the repo's
+rule, store errors are logged rather than returned
+(`apischedule.go:145-147`) — they carry bucket names and request ids.
+
+**R5 — Written for two readers.** A human skims; an agent parses. No
+`carlos` verb has a `--json` flag today, so this **introduces** one, and
+the fields are fixed here so the human and machine forms cannot drift:
+
+```
+copy: true
+as_of # time of the newest replicated write (§8)
+replicated_txid
+lease_held: bool # is the instance replicating right now (§6, §8)
+sync_class # stream|batched|daily|unknown — never defaulted
+verified # ok|failed|unverified
+```
+
+**R6 — Name what is not there.** A database declared in `data.dbs` but
+not replicated (§7) is listed as such. "The platform has no copy of
+`audit.db`" must be said before someone goes looking for it.
+
+## 6. Transport: the replica door
+
+**Decision: the console reads the region's data bucket directly.**
+Option B — having the owning box mirror replicas into a staging prefix
+in the deployment bucket — was considered and rejected. It needs a new
+box-side pass plus a sweeper, which contradicts §9's "no path that runs
+anything on the box"; its first-mirror latency is a CopyObject per LTX
+file; it lands tenant database bytes in the deployment bucket, which the
+moves design deliberately routed *around* (staging goes to a sibling
+data bucket, never the deployment bucket); and it does not even remove
+the data-bucket read, which `hibernate-verify` still needs. Its one
+advantage over A — cross-region reach — is a residency violation rather
+than a feature.
+
+### Configuration
+
+The console takes the same triple the agent reads —
+`CARLOS_HIBERNATE_BUCKET` / `_REGION` / `_PREFIX` — plus `CARLOS_REGION`
+for the placement compare, and opens one `*s3.Client` reusing
+`lease/endpoint.go`'s config path including `s3compat.Options()` and the
+endpoint override (`endpoint.go:110-131`).
+
+There is deliberately **no edges-file lookup** for another box's bucket.
+A bucket discovered that way is one IAM will refuse, so resolving it
+only converts a clear refusal into a confusing one.
+
+### Whether this needs an IAM change
+
+Probably not. The console opens S3 through the default credential chain
+(`cmd/console/main.go:394-399`) — the box's instance role, the same role
+the activator on that box uses to *write* `instances/<host>/`. If the
+console's box hosts any hibernating instance, the read authority is
+already there. If not, it is one read-only, in-region grant, smaller
+than the `iam:CreateUser`/`PutUserPolicy` the console already holds via
+`CARLOS_STORE_BACKEND=aws` (`objstore/aws.go:39-46`).
+
+### Resolution, and what is refused before S3 is touched
+
+`apps/<sqid>/<app>/instances/<host>.json` → `ValidHostShape`
+(`provision.go:250-262`, written as path-traversal defence — `path.Join`
+alone cleans `../x` to `x` rather than refusing it) → then three
+refusals, none of which make an S3 call:
+
+- `rec.Region != CARLOS_REGION` (both `""` in a single-region
+ deployment) → residency refusal naming the region.
+- a `fleet/<name>` region (`console/fleets.go:535`) → residency refusal;
+ the data is on the customer's own box. §13 records the v2.
+- `rec.Pool != ""` → refusal; a pooled record's data lives on the pool
+ box, the same class `movespass.go:150-153` already refuses.
+
+An `AccessDenied` from S3 maps to the residency refusal as a backstop,
+never to a generic 502.
+
+### The door
+
+Two handlers, one gate, one rate budget:
+
+- `list(app, instance, level)` → `[{level, min_txid, max_txid, size, created_at}]`
+- `get(app, instance, level, min_txid, max_txid)` with `Range` support
+
+Implement them as an HTTP adapter over litestream's own
+`lss3.ReplicaClient` **read methods**, with `Path` set per request. That
+way the server derives the key layout
+(`<prefix>/%04x/%016x-%016x.ltx`, `s3/replica_client.go:655-656`) from
+the same code the writer uses, and reader and writer cannot drift.
+
+Addressing by coordinate rather than key is what keeps `lock.json`
+— which sits under the same prefix and is not a member's to fetch —
+unreachable.
+
+Following the edge store door's F8 precedent (`storedoor.go:55-59`):
+bytes **stream through the console**, never a presigned URL, and nothing
+the request supplies may influence which bucket or client is used.
+
+Six wire specifics the implementation must honour:
+
+1. `list` accepts **levels 0–9 inclusive**. `CalcRestorePlan` asks for
+ `SnapshotLevel` (= 9, `compaction_level.go:9`) first, then 8 down to
+ 0 (`replica.go:1452, 1476-1485`). The activator writes only 0–3 and
+ 9; the door answers **empty** for the rest, never 404.
+2. Listings are ordered by `min_txid` **ascending** —
+ `restoreLevelCursor` assumes filename order, which S3 gives because
+ the names are zero-padded hex.
+3. `seek` and `useMetadata` are not on the wire. The client filters
+ `seek` locally; nothing in a non-timestamp restore reads `CreatedAt`
+ except debug logs.
+4. `size`, `min_txid` and `max_txid` are **mandatory**. `Restore`
+ rejects `Size < ltx.HeaderSize` (`replica.go:630`) and
+ `ResumableReader` uses size to detect a premature EOF
+ (`resumable_reader.go:96`). Checksums are not needed.
+5. `get` Range semantics match the S3 client exactly:
+ `bytes=<offset>-` when `size == 0`, else
+ `bytes=<offset>-<offset+size-1>` (`s3/replica_client.go:647-652`).
+ A missing object is a 404 the client maps to `os.ErrNotExist`.
+6. `ls` needs a third, **derived** field: `lease_held` (plus
+ `expires_at`), computed console-side from `lock.json`
+ (`leaser.go:31-35`; a released lease is deleted, `s3/leaser.go:180`).
+ The lock's **body is never returned** — `Owner` is the box's
+ identity. This is how §8's "is it replicating now" is answered
+ without exposing an object coordinate addressing deliberately hides.
+
+### Who may open it
+
+`PrincipalUser` only. Every service principal — the legacy app token
+(role `instance`) and every named service credential
+(`publish`/`instance`/`operate`/`admin`) — gets the same
+`http.NotFound` as a stranger.
+
+A 403 would be an oracle: it tells any token holder that a db door
+exists for this app. The gate ethos here is one byte-identical 404.
+
+No legitimate caller loses anything — an app reads its own database
+live, and CI has no business pulling production data. `admin` is refused
+**deliberately**, despite already reaching `store grant`.
+
+> **Revision, 2026-10-01 (implementation; Paul may overrule).** Service
+> principals get a 403, not the 404 above. The two door routes are
+> registered with the zero role Set, the repo's closed-door mechanism
+> since 2026-08-18, and the shared `cliRoute` gate answers every service
+> credential with one constant 403 ("closed to service credentials")
+> BEFORE any account or app lookup. The body is the same for every role,
+> account and app, so it reveals only that the door exists, which is
+> public in the code, and nothing about this app or account. Members who
+> are not in the account, and unknown apps and instances, still get the
+> byte-identical 404. A literal 404 would need either a role Set the
+> handler then overrides, or a change to the shared gate for every closed
+> door.
+
+Every successful open writes an `s.audit` row naming account, app, host,
+database and the replicated TXID served.
+
+### The read-only replica client
+
+The CLI implements litestream's `ReplicaClient` over this door, the way
+`internal/fleetapi` implements `release.Store` over the edge store door,
+and for the same reason: the member has no AWS credentials, by design.
+
+The interface has eight methods (`replica_client.go:19-50`). This
+implementation supplies the five a restore needs — `Type`, `Init`,
+`LTXFiles`, `OpenLTXFile`, `SetLogger` — and the three mutating ones
+(`WriteLTXFile`, `DeleteLTXFiles`, `DeleteAll`) return an error
+unconditionally.
+
+**The client cannot corrupt the replica, because it has no code that
+writes to one.** Not "is not supposed to" — has none. §12 records the
+verification and the three conditions it depends on.
+
+## 7. Which database — one, not eight
+
+`caps.Data` allows up to eight names (`maxDBs = 8`), but **a route
+replicates exactly one database**: `Route.DBPath` is single-valued and
+only `DBs[0]` is stamped (`host/caps_pass.go:294`, whose own adopt
+warning says so verbatim). The activator's `Replicate` takes one
+`dbPath`.
+
+So there is no `--db` flag and no selection story:
+
+- The replicated database is implied by the instance.
+- `--instance` selects among an app's instances — inferred when there is
+ one, required when there are several, because guessing hands someone
+ the wrong data while telling them it is the right data.
+- `ls` names the **declared-but-unreplicated** entries (`DBs[1:]`),
+ surfacing `dataAdvice`'s warning to a member who cannot read adopt
+ logs (R6).
+
+### `replicate: "app"` has no replica
+
+An app declaring `replicate: "app"` streams its own WAL and the platform
+replicates **nothing** for it (`caps.DataReplicateApp`; `caps.go:416`
+refuses `data.dbs` alongside it precisely to keep the two-replicators
+hazard unreachable).
+
+This must fail naming that cause. The dangerous failure is a generic "no
+replica found", which reads as transient and invites a retry loop; the
+operator needs to know the platform was never going to have this data.
+
+## 8. Age, and the three things that are not lag
+
+The client **cannot compute lag**. `ops.freshness` gets the live change
+counter from the source file on the box (`freshness.go:96-115`), which
+neither console nor CLI can reach. Any claimed distance from live would
+be invented.
+
+What the replica alone supports:
+
+1. **Age of the newest replicated write.** Take it from the **LTX
+ header timestamp**, not S3 `LastModified` — a compacted file's
+ `LastModified` is compaction time, and the `ltx` compactor carries
+ `maxHdr.Timestamp` through (`compactor.go:118`). One ranged GET of
+ `ltx.HeaderSize` bytes on the max-TXID file gives the true
+ newest-write time at any level. Using `LastModified` would report an
+ L3 compaction as a fresh write — a stale copy looking current, the
+ exact failure this command exists to prevent.
+2. **Whether it is replicating now** — `lease_held`, derived from
+ `lock.json` (§6).
+3. **The expected bound — when it is knowable.** `Replication` lives on
+ `registry.Route` (`registry.go:126`) in the **box** registry;
+ `provision.Record` has no such field. So the console knows the class
+ only for routes on its own box. For an in-region sibling it is
+ genuinely unknown, and R2 requires saying so rather than defaulting.
+ §13 records the additive follow-up.
+4. **The drill verdict** — `dbverify`, per host, hibernation drill only.
+
+The first draft claimed "no explicit sync interval, so 1s default,
+therefore seconds behind live". That is wrong for the instances this
+targets, and wrong in the dangerous direction: classes are `""`/`stream`
+≈ 1s, `batched` 5m, `daily` 24h (`activator.go:205-230`,
+`lease.go:245-252`). An awake `daily` instance is **legitimately up to a
+day behind**, and reporting its age without its bound makes correct
+platform behaviour look like current data.
+
+Retention differs between the two replicators too: `SnapshotInterval`
+24h / `SnapshotRetention` 7d (`ops/litestream.go:64-65`) reach only
+`/etc/litestream.yml`. The activator's embedded store never sets
+`Retention`, so hibernating replicas run on litestream's
+`DefaultRetention` of **24h**.
+
+### A sleeping instance is consistent, not necessarily complete
+
+`stop()` logs and swallows a failed final checkpoint
+(`activator.go:876-881`) — the hole the instance-move spec had to add a
+re-checkpoint for. A parked replica is consistent at a *committed*
+boundary (LTX carries committed frames only) but may trail the sleep
+point.
+
+So: "consistent as of its last successful sync, which on a clean sleep
+is the sleep point" — never "complete as of when it went to sleep".
+`carlos restart` and Bounce go through the same `stop()`.
+
+## 9. Security posture
+
+**The exposure is real and is the point of the gate.** This command
+moves production data onto a laptop. Read-only does not reduce that; a
+copy leaks exactly as well as an original. Hence: members only, no
+service principals of any role, an audit row per open, `--out` required
+and refused if it exists, 0600 on anything kept, a 0700 private
+directory for anything temporary.
+
+**Nothing new is reachable on the box, and nothing new runs there.** No
+door on the tenant process, no new agent verb, no pass in the converge
+loop. The sandbox (`DynamicUser`, `ProtectSystem=strict`, empty
+`CapabilityBoundingSet`) is untouched, and the same-uid sibling residual
+in `activator/admintoken.go:8-12` is not widened, because no part of
+this design runs on a box at all. Option B would have forfeited this
+sentence; A keeps it.
+
+**One new read authority:** the console gains in-region, read-only
+access to the data bucket's `instances/*`. Bounded by residency — it
+cannot reach another region's data even if asked, and §6 refuses before
+trying rather than surfacing an `AccessDenied`.
+
+**The failure mode to design against is not corruption, it is
+confidence.** Every serious way this command hurts someone routes
+through a caller believing old data is current, or believing a write
+landed. §5 and §4's read-only enforcement exist for that; they are not
+decoration on the security story, they are most of it.
+
+## 10. Failure matrix
+
+Each row is a test.
+
+| Condition | Result |
+|---|---|
+| Not a member of the account | Byte-identical 404, matching `instanceGate` |
+| Any service principal (app token, or `publish`/`instance`/`operate`/`admin`) | Same byte-identical 404 — never a 403 |
+| No such app | Byte-identical 404 |
+| `replicate: "app"` | Refusal naming that cause specifically (§7) |
+| Several instances, no `--instance` | Refusal listing the hosts |
+| Instance in another region | Residency refusal naming the region — no S3 call |
+| Instance region `fleet/<name>` | Residency refusal — no S3 call (§13 records the v2) |
+| `rec.Pool != ""` | Refusal — data lives on the pool box |
+| S3 `AccessDenied` | Mapped to the residency refusal, never a generic 502 |
+| Declared-but-unreplicated db named | Refusal naming it as never replicated (§7) |
+| Nothing replicated yet | "Nothing replicated yet" — match **both** `ErrNoSnapshots` and `ErrTxNotAvailable` (`lease.go:394`) |
+| `list` for a level the activator never writes | Empty list, **not** 404 (§6.1) |
+| Replica changed under the restore | One retry around the whole `Restore` (§12) |
+| Restore fails integrity check | Refuse to hand over the file; surface the `dbverify` verdict |
+| Instance asleep | Succeeds; reports age, `lease_held: false`, class or unknown |
+| Instance awake on `daily` class | Succeeds; age reported beside its 24h bound (R2) |
+| Class not knowable (sibling box) | `sync_class: "unknown"`, never `stream` (R2) |
+| Write attempted in `shell`/`query` | SQLite `attempt to write a readonly database`, surfaced plainly |
+| `--out` omitted, or already exists | Usage error |
+| Host failing `ValidHostShape` | Refused before it becomes a key segment |
+| Client asked to write a replica | Unconditional error — no code path exists |
+
+## 11. Testing shape
+
+Following `apischedule_test.go` and `internal/lease/check_test.go`:
+
+- **Door:** the full gate matrix, plus a coordinate-confinement test
+ asserting `lock.json` and any key outside the resolved prefix are
+ unreachable. That test is what keeps this from becoming an object
+ proxy as the code ages.
+- **Wire conformance:** levels 0–9 answer empty rather than 404;
+ listings ascend by `min_txid`; `size`/`min_txid`/`max_txid` always
+ present; Range semantics match `s3/replica_client.go:647-652` for both
+ the `size == 0` and sized forms.
+- **Replica client, both halves:** the three mutating methods error
+ without touching the network **and** a call-counting fake asserts they
+ stay at zero across a full restore. The second half is what catches a
+ litestream upgrade changing the restore path.
+- **`ReplicaClientV3` negative assertion** — §12 condition (i).
+- **Compile-time** `var _ litestream.ReplicaClient = (*client)(nil)`.
+- **Freshness:** a table over (age, `lease_held`, class-or-unknown,
+ verdict) asserting the wording distinguishes asleep from failing,
+ always carries the bound or admits it is unknown, and **never**
+ renders unknown as `stream`.
+- **Age source:** a compacted L3 file whose `LastModified` is far newer
+ than its LTX header timestamp reports the header time (§8.1).
+- **Selection:** `caps.Data` fixtures for `DBs[1:]` unreplicated and the
+ `replicate: "app"` refusal.
+- **Read-only:** an `UPDATE` in `query` fails with the pinned error
+ string, and the file on disk is byte-identical afterwards.
+- **Permissions:** the temp directory is 0700, `--out` lands 0600, and
+ `pull` succeeds when temp and destination are on different
+ filesystems.
+
+Definition of done includes a `docs/site/cli.md` heading — the
+`internal/docsite` gate fails without one (AGENTS.md).
+
+## 12. Why the safety claim holds
+
+Verified against litestream v0.5.14 during review, because §6's central
+claim depends on it.
+
+`Replica.Restore` (`replica.go:544-720`) → `CalcRestorePlan`
+(`replica.go:1485ff`, `LTXFiles` per level) → `NewResumableReader`,
+whose client parameter is the narrow `LTXFileOpener` interface
+(`internal/resumable_reader.go:51,72`) — `OpenLTXFile` only.
+`WriteLTXFile`, `DeleteLTXFiles` and `DeleteAll` are reached only from
+`Sync`, `EnforceRetention`, and `db.go`'s snapshot/retention paths
+(`replica.go:192,312`; `db.go:2470,2526,2649`), none of which run
+without a `*DB`/`Store`. With a nil `*DB`, they are unreachable.
+
+Three conditions this depends on. Each is a test in §11:
+
+1. **The client must not implement `ReplicaClientV3`** (`v3.go:145`).
+ `Restore` type-asserts for it (`replica.go:612`) and would take the
+ v0.3 path instead.
+2. **Never set `RestoreOptions.Follow`.** It writes into the local
+ output and a `-txid` sidecar, and keeps polling.
+3. **Run `lease.CheckDB` after the restore** rather than relying on
+ `opt.IntegrityCheck`, matching `lease.Restore`'s existing pattern.
+
+### What the claim does *not* cover
+
+`check.go`'s reasoning is about not *shadowing the writer*. It does not
+promise the reader a stable listing. `CalcRestorePlan` lists every level
+once and opens lazily, so the writer's L0 retention
+(`DefaultL0Retention` 5m, checked hourly by the activator,
+`lease.go:78`) can delete a planned file before it is opened.
+
+The outcome is a **failed** restore, never a torn one: the output is
+`<out>.tmp`, renamed only on success (`replica.go:661-663`).
+
+The retry must wrap the **whole `Restore`**, not an individual read:
+`ResumableReader` treats `ErrNotExist` as fatal and does not retry
+(`resumable_reader.go:72-75`), so a vanished planned file needs a fresh
+plan, which only a new `Restore` produces.
+
+## 13. Open questions and follow-ups
+
+1. **Fleet-homed instances (v2).** §6 refuses them: their data is in the
+ customer's bucket and IAM will not cross. The reverse tunnel
+ (`edge/tunnel.go`) is a real synchronous edge→box path for exactly
+ the hosts that dial in — so the v2 is the box serving its own replica
+ coordinates over its tunnel channel, with the console never touching
+ the customer's bucket. Different transport, same wire shape.
+2. **An additive `replication` field on the instance record.** Closes
+ §8's `sync_class: "unknown"` for in-region siblings. Wire-additive
+ and therefore allowed; worth doing regardless of this command.
+3. **`shell`'s prompt.** An embedded REPL is small code but a real
+ surface (history, formatting, dot commands). Ship `ls`/`pull`/`query`
+ first, then `shell`.
+4. **Console web view.** Out of scope, but §6's door is what it would
+ use; confirm the coordinate-addressed shape does not foreclose it.
+5. **Past points in time.** Not offered. Given the activator's effective
+ 24h `DefaultRetention` (§8), is "current replica only" the right
+ permanent scope?
+6. **Rate limiting.** Pulls are large; neither the token-keyed instance
+ burst nor the address-keyed write budget fits. Needs its own budget.
+
+### Decomposition
+
+Three units, in order: (1) door + read-only client + `pull` + `query`;
+(2) `ls` and its freshness fields; (3) `shell`.
+
+## 14. What review changed
+
+**Round one.** The safety claim survived verification (§12). The
+transport did not: "the console serves objects it already stores" was
+false. Also corrected — a route replicates exactly one database, not up
+to eight; `daily` class is legitimately 24h behind, not "seconds behind
+live"; a 403 for service principals was an oracle; litestream is a
+dependency, not vendored; `FleetDBs` is the complement of the
+activator's set; the console has no registry snapshot; `dbverify` is
+keyed by host; the mode-0600 temp file was not constructible.
+
+**Round two.** Option A chosen and B rejected on its merits (§6), so §9
+regains "nothing runs on the box". The edges-file lookup was dropped as
+buying nothing IAM will allow. Six wire specifics added, including
+levels 0–9 answering empty rather than 404 and mandatory `size`. Two new
+findings, both in the direction of overstating freshness:
+
+- **`sync_class` is unknowable off-box** — `Replication` is on
+ `registry.Route`, not `provision.Record`. R2 now requires an explicit
+ "unknown" that must never render as `stream`.
+- **Age must come from the LTX header timestamp**, not S3
+ `LastModified`, or a compaction reads as a fresh write.
+
+Plus `lease_held` as a derived `ls` field (answering "is it replicating"
+without exposing `lock.json`), the retry wrapping the whole `Restore`,
+and `pull` copying rather than renaming across filesystems.
diff --git a/internal/console/apidb.go b/internal/console/apidb.go
new file mode 100644
index 000000000..bf1a783d0
--- /dev/null
+++ b/internal/console/apidb.go
@@ -0,0 +1,471 @@
+// The `carlos db` replica door (spec 2026-09-02 §6). It serves an
+// instance's replicated LTX files BY COORDINATE (level plus TXID span) from
+// the region data bucket, to a member of the account and nobody else, so the
+// CLI can restore a private copy on the member's own machine. The console
+// never executes tenant SQL and never holds a restored database.
+//
+// resolveReplica is the half that makes no S3 call: resolving an instance
+// record to one replica prefix, and refusing everything that is not this
+// console's to read. Every refusal there happens BEFORE the bucket is
+// touched. That is deliberate: a door that tried and mapped AccessDenied
+// would leak the attempt, the latency, and eventually a bucket name into a
+// log a member can provoke.
+
+package console
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "log"
+ "net/http"
+ "os"
+ "sort"
+ "strconv"
+ "sync"
+ "time"
+
+ "github.com/aws/smithy-go"
+ "github.com/superfly/ltx"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/dbdoor"
+ "github.com/carlosframework/platform/internal/lease"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+)
+
+var (
+ // ErrResidency: the data is in another region's bucket, or on a
+ // customer's own fleet box. IAM makes those reads impossible by design,
+ // so this is the design working rather than a fault to retry.
+ ErrResidency = errors.New("db: the instance's data is outside this region")
+ // ErrPooled: a pooled record's data lives on the pool box, the same
+ // class the instance-move worker refuses.
+ ErrPooled = errors.New("db: pooled instances have no replica of their own")
+ // ErrBadHost: path.Join cleans "../x" to "x" rather than refusing it,
+ // so a host becomes a key segment only after ValidHostShape, and only
+ // when it is the host the record itself names.
+ ErrBadHost = errors.New("db: invalid host")
+)
+
+func (s *Server) resolveReplica(rec provision.Record, host string) (string, error) {
+ if !provision.ValidHostShape(host) || rec.Host != host {
+ return "", fmt.Errorf("%w: %q", ErrBadHost, host)
+ }
+ if rec.Pool != "" {
+ return "", fmt.Errorf("%w: %s is in pool %q", ErrPooled, host, rec.Pool)
+ }
+ // A fleet placement is checked on its own, not left to the region
+ // compare: the data is on the customer's box whatever this console's
+ // CARLOS_REGION says.
+ if _, fleet := registry.FleetPlacement(rec.Region); fleet {
+ return "", fmt.Errorf("%w: %s is placed on fleet %q", ErrResidency, host, rec.Region)
+ }
+ if rec.Region != s.cfg.Region {
+ return "", fmt.Errorf("%w: %s is homed in %q, this console serves %q",
+ ErrResidency, host, rec.Region, s.cfg.Region)
+ }
+ return activator.HibernatePrefix(s.cfg.HibernatePrefix, host), nil
+}
+
+// replicaReader is the only part of a litestream replica client the door
+// may hold: the two read methods. Holding the narrow interface means no
+// handler here can reach WriteLTXFile, DeleteLTXFiles or DeleteAll by
+// mistake, whatever the concrete client underneath can do.
+type replicaReader interface {
+ LTXFiles(ctx context.Context, level int, seek ltx.TXID, useMetadata bool) (ltx.FileIterator, error)
+ OpenLTXFile(ctx context.Context, level int, minTXID, maxTXID ltx.TXID, offset, size int64) (io.ReadCloser, error)
+}
+
+// cliDBBurst bounds the db door per TOKEN within the ordinary minute window.
+// Its own budget, not the instance verbs' 60 (spec §13.6): one restore is
+// ten listings plus one request per planned file, and an app with a long L0
+// tail plans dozens. The number is a ceiling on a stolen token, not a
+// policing of honest pulls.
+const cliDBBurst = 600
+
+// dbAuditAction names the audit row every served file writes.
+const dbAuditAction = "db-read"
+
+// dbRequest is what dbGate resolves for both handlers.
+type dbRequest struct {
+ acct *Account
+ actor string
+ app string
+ host string
+ prefix string
+ rd replicaReader
+}
+
+// dbGate is the door's whole gate, in the repo's order: principal (cliRoute
+// has already refused every service credential at the zero Set) ->
+// membership (byte-identical 404) -> budget (429) -> configuration (503) ->
+// the record (404 when absent) -> the class rail and residency (409), none
+// of which touches the bucket -> the replica client. ok=false means a
+// response has been written.
+func (s *Server) dbGate(w http.ResponseWriter, r *http.Request) (dbRequest, bool) {
+ // Belt to cliRoute's braces: the zero Set already stops every service
+ // principal before this runs, so this line is only reached by a request
+ // that did not come through cliRoute at all.
+ if p, ok := principalFrom(r.Context()); !ok || p.Kind != PrincipalUser {
+ http.NotFound(w, r)
+ return dbRequest{}, false
+ }
+ actor, acct, app, ok := s.apiAccountApp(w, r)
+ if !ok {
+ return dbRequest{}, false
+ }
+ if !s.cli.db.allow(hashToken(bearer(r)), time.Now()) {
+ jsonError(w, http.StatusTooManyRequests, "too many database requests. Wait a minute and try again.")
+ return dbRequest{}, false
+ }
+ if s.cfg.Releases == nil || s.cfg.HibernateBucket == "" {
+ jsonError(w, http.StatusServiceUnavailable, "this console has no data bucket set up, so it cannot read databases.")
+ return dbRequest{}, false
+ }
+ host := r.PathValue("host")
+ if !provision.ValidHostShape(host) {
+ http.NotFound(w, r)
+ return dbRequest{}, false
+ }
+ ctx := r.Context()
+ rec, err := provision.GetRecord(ctx, s.cfg.Releases, acct.Sqid, app, host)
+ if errors.Is(err, os.ErrNotExist) {
+ http.NotFound(w, r)
+ return dbRequest{}, false
+ }
+ if err != nil {
+ log.Printf("console: api db %s/%s/%s: read record: %v", acct.Sqid, app, host, err)
+ jsonError(w, http.StatusBadGateway, "could not read the instance. Try again.")
+ return dbRequest{}, false
+ }
+ if msg := dbClassRefusal(*rec); msg != "" {
+ jsonError(w, http.StatusConflict, msg)
+ return dbRequest{}, false
+ }
+ prefix, err := s.resolveReplica(*rec, host)
+ switch {
+ case errors.Is(err, ErrBadHost):
+ http.NotFound(w, r)
+ return dbRequest{}, false
+ case errors.Is(err, ErrPooled):
+ jsonError(w, http.StatusConflict, fmt.Sprintf(
+ "%s runs in the pool %s, so its data is on the pool's own server. carlos db cannot read it.", host, rec.Pool))
+ return dbRequest{}, false
+ case errors.Is(err, ErrResidency):
+ jsonError(w, http.StatusConflict, s.dbResidencyRefusal(*rec))
+ return dbRequest{}, false
+ case err != nil:
+ log.Printf("console: api db %s/%s/%s: resolve: %v", acct.Sqid, app, host, err)
+ jsonError(w, http.StatusBadGateway, "could not read the replica. Try again.")
+ return dbRequest{}, false
+ }
+ rd, err := s.dbReplica(ctx, prefix)
+ if err != nil {
+ log.Printf("console: api db %s/%s/%s: replica client: %v", acct.Sqid, app, host, err)
+ jsonError(w, http.StatusBadGateway, "could not read the replica. Try again.")
+ return dbRequest{}, false
+ }
+ return dbRequest{acct: acct, actor: actor, app: app, host: host, prefix: prefix, rd: rd}, true
+}
+
+// dbClassRefusal is the class rail as a sentence, or "" for an
+// activator-managed exec instance: the only class whose database the
+// platform replicates (spec §1 scope). Same three facts the move doors
+// refuse on; the fleet and pool cases are resolveReplica's.
+func dbClassRefusal(rec provision.Record) string {
+ if k := rec.Kind; k != "" && k != "instance" {
+ return fmt.Sprintf("%s is a %s route. The platform keeps a database copy only for instances.", rec.Host, k)
+ }
+ if rec.Addr != "" || len(rec.Addrs) > 0 {
+ return fmt.Sprintf("%s is served from its own address, so the platform does not manage its database.", rec.Host)
+ }
+ if !rec.HibernateOrDefault() {
+ return fmt.Sprintf("%s does not sleep, so the platform does not keep a copy of its database.", rec.Host)
+ }
+ return ""
+}
+
+// dbResidencyRefusal names where the data is, in the vocabulary the
+// regions door uses (the home region by its CONSOLE_HOME_REGION name).
+func (s *Server) dbResidencyRefusal(rec provision.Record) string {
+ if fleet, ok := registry.FleetPlacement(rec.Region); ok {
+ return fmt.Sprintf("%s runs on your fleet %s, so its data stays on that fleet's own server. carlos db cannot read it.", rec.Host, fleet)
+ }
+ return fmt.Sprintf("%s keeps its data in %s. This console can only read data kept in %s.",
+ rec.Host, s.dbRegionName(rec.Region), s.dbRegionName(s.cfg.Region))
+}
+
+func (s *Server) dbRegionName(region string) string {
+ if r := s.homeRegionOut(region); r != "" {
+ return "region " + r
+ }
+ return "the home region"
+}
+
+// dbStoreFailure answers a failed bucket read. AccessDenied is the
+// residency refusal as a backstop (spec §6): the role this console runs as
+// cannot read that data, which is the design working, not a fault to
+// retry. Everything else is logged with its detail (bucket names, request
+// ids) and answered with none of it.
+func (s *Server) dbStoreFailure(w http.ResponseWriter, g dbRequest, what string, err error) {
+ log.Printf("console: api db %s/%s/%s: %s: %v", g.acct.Sqid, g.app, g.host, what, err)
+ var api smithy.APIError
+ if errors.As(err, &api) && api.ErrorCode() == "AccessDenied" {
+ jsonError(w, http.StatusConflict, fmt.Sprintf(
+ "this console is not allowed to read the data for %s. It is kept outside the region this console serves.", g.host))
+ return
+ }
+ jsonError(w, http.StatusBadGateway, "could not read the replica. Try again.")
+}
+
+// parseDBLevel accepts a bare decimal level in 0..MaxLevel. Anything else
+// is not a coordinate.
+func parseDBLevel(s string) (int, bool) {
+ n, err := strconv.ParseUint(s, 10, 8)
+ if err != nil || n > dbdoor.MaxLevel {
+ return 0, false
+ }
+ return int(n), true
+}
+
+// handleAPIDBList answers one level's listing. An out-of-range level is a
+// 404 (it is not a coordinate); a level with nothing in it is 200 and an
+// empty list (it is a coordinate that happens to be empty, spec §6.1).
+func (s *Server) handleAPIDBList(w http.ResponseWriter, r *http.Request) {
+ level, ok := parseDBLevel(r.PathValue("level"))
+ if !ok {
+ http.NotFound(w, r)
+ return
+ }
+ g, ok := s.dbGate(w, r)
+ if !ok {
+ return
+ }
+ itr, err := g.rd.LTXFiles(r.Context(), level, 0, false)
+ if err != nil {
+ s.dbStoreFailure(w, g, fmt.Sprintf("list level %d", level), err)
+ return
+ }
+ out := dbdoor.ListResponse{Files: []dbdoor.LTXFile{}}
+ for itr.Next() {
+ f := itr.Item()
+ out.Files = append(out.Files, dbdoor.LTXFile{
+ Level: level, MinTXID: uint64(f.MinTXID), MaxTXID: uint64(f.MaxTXID),
+ Size: f.Size, CreatedAt: f.CreatedAt,
+ })
+ }
+ err = itr.Err()
+ if cerr := itr.Close(); err == nil {
+ err = cerr
+ }
+ if err != nil {
+ s.dbStoreFailure(w, g, fmt.Sprintf("list level %d", level), err)
+ return
+ }
+ sort.Slice(out.Files, func(i, j int) bool { return out.Files[i].MinTXID < out.Files[j].MinTXID })
+ writeJSON(w, http.StatusOK, out)
+}
+
+// handleAPIDBGet streams one LTX file by coordinate. The bytes pass through
+// the console, never a presigned URL (the edge store door's F8 precedent):
+// nothing the request supplies picks the bucket or the key, only a level
+// and a span inside the prefix resolveReplica chose.
+func (s *Server) handleAPIDBGet(w http.ResponseWriter, r *http.Request) {
+ level, ok := parseDBLevel(r.PathValue("level"))
+ if !ok {
+ http.NotFound(w, r)
+ return
+ }
+ minTXID, maxTXID, err := dbdoor.ParseSpan(r.PathValue("span"))
+ if err != nil {
+ http.NotFound(w, r)
+ return
+ }
+ g, ok := s.dbGate(w, r)
+ if !ok {
+ return
+ }
+ rangeHdr := r.Header.Get("Range")
+ offset, size, err := dbdoor.ParseRange(rangeHdr)
+ if err != nil {
+ jsonError(w, http.StatusRequestedRangeNotSatisfiable, "unsupported range")
+ return
+ }
+ if !s.dbStreams.acquire(g.actor) {
+ w.Header().Set("Retry-After", dbStreamRetryAfter)
+ jsonError(w, http.StatusTooManyRequests, "too many database downloads at once. Wait a few seconds and try again.")
+ return
+ }
+ defer s.dbStreams.release(g.actor)
+ // The idle watchdog: any stretch of dbStreamIdle with no byte moved
+ // cancels the bucket read. The write side has its own deadline below,
+ // renewed per chunk, because a cancelled context does not unblock a
+ // write to a client that has stopped reading.
+ idle := s.dbStreamIdle
+ ctx, cancel := context.WithCancel(r.Context())
+ defer cancel()
+ watchdog := time.AfterFunc(idle, cancel)
+ defer watchdog.Stop()
+ rc, err := g.rd.OpenLTXFile(ctx, level, ltx.TXID(minTXID), ltx.TXID(maxTXID), offset, size)
+ if errors.Is(err, os.ErrNotExist) {
+ http.NotFound(w, r)
+ return
+ }
+ if err != nil {
+ s.dbStoreFailure(w, g, fmt.Sprintf("open level %d %s", level, dbdoor.Span(minTXID, maxTXID)), err)
+ return
+ }
+ defer rc.Close()
+ // One row per successful GET, ranged or not. A ranged read is still a
+ // read of the file: skipping it would let a caller list the coordinates
+ // and fetch every file from byte 1 with no row written. A resumed read
+ // says where it started.
+ detail := fmt.Sprintf("read database replica file level %d, transactions %d to %d (cli)", level, minTXID, maxTXID)
+ if offset > 0 {
+ detail = fmt.Sprintf("read database replica file level %d, transactions %d to %d, from byte %d (cli)", level, minTXID, maxTXID, offset)
+ }
+ s.auditActivity(g.acct, g.app, g.actor, Activity{
+ Action: dbAuditAction, TargetType: "instance", Target: g.host, Detail: detail,
+ })
+ w.Header().Set("Content-Type", "application/octet-stream")
+ w.Header().Set("Cache-Control", "no-store")
+ status := http.StatusOK
+ if rangeHdr != "" {
+ status = http.StatusPartialContent
+ }
+ rw := http.NewResponseController(w)
+ // The deadline is never cleared here. net/http flushes what is still
+ // buffered (the last bytes, the chunked terminator) in finishRequest,
+ // after this handler returns, and clears the write deadline itself
+ // only after that flush. Clearing it on the way out would leave that
+ // flush unbounded, holding the connection for as long as a client that
+ // stopped reading near the end cares to wait.
+ _ = rw.SetWriteDeadline(time.Now().Add(idle))
+ w.WriteHeader(status)
+ err = dbStream(w, rw, rc, watchdog, idle)
+ // Flush the tail now, under a fresh deadline, so it is done (or cut)
+ // before the deferred release gives this stream's slot back. What
+ // finishRequest has left after this is the chunked terminator, still
+ // under the same deadline. On a failed read the deadline is renewed
+ // too, so the client gets its truncated body promptly (and resumes)
+ // rather than a connection cut by a deadline the stall used up.
+ _ = rw.SetWriteDeadline(time.Now().Add(idle))
+ if ferr := rw.Flush(); err == nil {
+ err = ferr
+ }
+ if err != nil {
+ // The status is already on the wire. The client's resumable reader
+ // sees a short body against the listed size and reopens at the
+ // offset it reached.
+ log.Printf("console: api db %s/%s/%s: stream level %d %s: %v",
+ g.acct.Sqid, g.app, g.host, level, dbdoor.Span(minTXID, maxTXID), err)
+ }
+}
+
+// dbStream copies rc to w a chunk at a time, renewing the write deadline
+// before each chunk and the read watchdog after it, so a stream ends when
+// either side stops moving for idle.
+func dbStream(w io.Writer, rw *http.ResponseController, rc io.Reader, watchdog *time.Timer, idle time.Duration) error {
+ buf := make([]byte, 32<<10)
+ for {
+ n, rerr := rc.Read(buf)
+ if n > 0 {
+ if err := rw.SetWriteDeadline(time.Now().Add(idle)); err != nil && !errors.Is(err, http.ErrNotSupported) {
+ return err
+ }
+ if _, err := w.Write(buf[:n]); err != nil {
+ return err
+ }
+ watchdog.Reset(idle)
+ }
+ if rerr == io.EOF {
+ return nil
+ }
+ if rerr != nil {
+ return rerr
+ }
+ }
+}
+
+// dbReplicaCacheMax bounds the replica-client cache. Each client owns an
+// HTTP transport, so one per request would cost a TLS handshake per file and
+// leave an idle connection behind each; one per prefix lets a restore's
+// dozens of requests share one. Over the bound the map is dropped
+// wholesale, the rateLimiter's rule: forgetting clients costs a handshake,
+// a walk would cost a lock held across every door.
+const dbReplicaCacheMax = 64
+
+type dbReplicaCache struct {
+ mu sync.Mutex
+ clients map[string]replicaReader
+}
+
+// cachedReadReplica is the production dbReplica: lease.ReadReplicaClient on
+// the configured bucket, one per prefix.
+func (s *Server) cachedReadReplica(ctx context.Context, prefix string) (replicaReader, error) {
+ c := &s.dbClients
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ if rd, ok := c.clients[prefix]; ok {
+ return rd, nil
+ }
+ rc, err := lease.ReadReplicaClient(ctx, s.cfg.HibernateBucket, prefix, s.cfg.HibernateRegion)
+ if err != nil {
+ return nil, err
+ }
+ if c.clients == nil || len(c.clients) >= dbReplicaCacheMax {
+ c.clients = make(map[string]replicaReader, 8)
+ }
+ c.clients[prefix] = rc
+ return rc, nil
+}
+
+// The db door's stream bounds. The per-minute token budget bounds how many
+// requests start, not how many are still running, and a slow or stalled
+// download holds a goroutine, an S3 connection and a client connection for
+// as long as it lasts. So active file streams are bounded per principal and
+// across the console, and refused with 429 and Retry-After when full; and
+// each stream must move a byte at least every dbStreamIdle, in both
+// directions, or it is cut. The CLI's restore downloads one file at a time
+// (dbdoor.RestoreTo), so a person's restore needs one slot.
+const (
+ dbMaxStreams = 32
+ dbMaxStreamsPerPrincipal = 4
+ dbStreamIdleDefault = 30 * time.Second
+ dbStreamRetryAfter = "5"
+)
+
+type dbStreamLimits struct {
+ mu sync.Mutex
+ maxTotal, maxPer int
+ total int
+ per map[string]int
+}
+
+func newDBStreamLimits(maxTotal, maxPer int) *dbStreamLimits {
+ return &dbStreamLimits{maxTotal: maxTotal, maxPer: maxPer, per: map[string]int{}}
+}
+
+// acquire takes a slot for key, or reports that the console or key is full.
+func (l *dbStreamLimits) acquire(key string) bool {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ if l.total >= l.maxTotal || l.per[key] >= l.maxPer {
+ return false
+ }
+ l.total++
+ l.per[key]++
+ return true
+}
+
+func (l *dbStreamLimits) release(key string) {
+ l.mu.Lock()
+ defer l.mu.Unlock()
+ l.total--
+ if l.per[key]--; l.per[key] <= 0 {
+ delete(l.per, key)
+ }
+}
diff --git a/internal/console/apidb_test.go b/internal/console/apidb_test.go
new file mode 100644
index 000000000..bf7501a20
--- /dev/null
+++ b/internal/console/apidb_test.go
@@ -0,0 +1,862 @@
+package console
+
+import (
+ "context"
+ "database/sql"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "net"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/aws/smithy-go"
+ lsfile "github.com/benbjohnson/litestream/file"
+ lss3 "github.com/benbjohnson/litestream/s3"
+ "github.com/superfly/ltx"
+
+ "github.com/carlosframework/platform/internal/dbdoor"
+ "github.com/carlosframework/platform/internal/dbdoor/dbdoortest"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/svcrole"
+)
+
+// Every one of these is refused BEFORE any S3 call. A door that reached the
+// bucket and mapped AccessDenied would leak both latency and the fact that
+// a bucket was tried. resolveReplica takes no store at all, so it cannot.
+func TestResolveReplicaRefusesBeforeTouchingS3(t *testing.T) {
+ s := &Server{}
+ s.cfg.Region = "eu-west-1"
+ s.cfg.HibernatePrefix = "instances"
+
+ cases := []struct {
+ name string
+ rec provision.Record
+ host string
+ want error
+ }{
+ {
+ name: "another region",
+ rec: provision.Record{Host: "a.bab.oncarlos.com", Region: "us-east-1"},
+ host: "a.bab.oncarlos.com",
+ want: ErrResidency,
+ },
+ {
+ name: "the home region seen from a regional console",
+ rec: provision.Record{Host: "a.bab.oncarlos.com"},
+ host: "a.bab.oncarlos.com",
+ want: ErrResidency,
+ },
+ {
+ name: "fleet-homed",
+ rec: provision.Record{Host: "a.bab.oncarlos.com", Region: "fleet/acme"},
+ host: "a.bab.oncarlos.com",
+ want: ErrResidency,
+ },
+ {
+ name: "pooled",
+ rec: provision.Record{Host: "a.bab.oncarlos.com", Region: "eu-west-1", Pool: "shared"},
+ host: "a.bab.oncarlos.com",
+ want: ErrPooled,
+ },
+ {
+ name: "hostile host",
+ rec: provision.Record{Host: "../../etc", Region: "eu-west-1"},
+ host: "../../etc",
+ want: ErrBadHost,
+ },
+ {
+ name: "record for another host",
+ rec: provision.Record{Host: "b.bab.oncarlos.com", Region: "eu-west-1"},
+ host: "a.bab.oncarlos.com",
+ want: ErrBadHost,
+ },
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ _, err := s.resolveReplica(tc.rec, tc.host)
+ if !errors.Is(err, tc.want) {
+ t.Fatalf("resolveReplica = %v, want %v", err, tc.want)
+ }
+ })
+ }
+}
+
+// A fleet placement is refused even on a console whose own region string
+// happened to match it: the data is on the customer's box, whatever a
+// misconfigured CARLOS_REGION says.
+func TestResolveReplicaRefusesAFleetEvenWhenTheRegionMatches(t *testing.T) {
+ s := &Server{}
+ s.cfg.Region = "fleet/acme"
+ s.cfg.HibernatePrefix = "instances"
+ if _, err := s.resolveReplica(provision.Record{Host: "a.bab.oncarlos.com", Region: "fleet/acme"}, "a.bab.oncarlos.com"); !errors.Is(err, ErrResidency) {
+ t.Fatalf("resolveReplica = %v, want ErrResidency", err)
+ }
+}
+
+// The happy path is an in-region, non-pooled instance, and the prefix is
+// the activator's own layout, not a string this package invents.
+func TestResolveReplicaBuildsTheActivatorPrefix(t *testing.T) {
+ s := &Server{}
+ s.cfg.Region = "eu-west-1"
+ s.cfg.HibernatePrefix = "instances"
+ got, err := s.resolveReplica(
+ provision.Record{Host: "a.bab.oncarlos.com", Region: "eu-west-1"},
+ "a.bab.oncarlos.com")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if got != "instances/a.bab.oncarlos.com" {
+ t.Fatalf("prefix = %q, want instances/a.bab.oncarlos.com", got)
+ }
+}
+
+// A single-region deployment leaves both empty, and "" == "" must be a
+// match rather than a refusal, otherwise the door never works anywhere
+// that has not set CARLOS_REGION. Home-region records keep region "" too
+// (Config.HomeRegion), so this is also the flagship's home console.
+func TestResolveReplicaTreatsEmptyRegionsAsAMatch(t *testing.T) {
+ s := &Server{}
+ s.cfg.HibernatePrefix = "instances"
+ if _, err := s.resolveReplica(
+ provision.Record{Host: "a.bab.oncarlos.com"}, "a.bab.oncarlos.com"); err != nil {
+ t.Fatalf("single-region deployment refused: %v", err)
+ }
+}
+
+// ---- the door handlers (Task 4) ----------------------------------------------
+
+const dbTestHost = "acme.go.tito.io"
+
+type dbFx struct {
+ *consoleFixture
+ member string
+ root string // the fake data bucket: one file replica per prefix
+ opens *atomic.Int32 // how many times the door reached for a replica client
+}
+
+// dbFixture is an in-region, hibernating exec instance of titogo whose
+// replica is a real litestream replica on disk, served through the door's
+// replica seam exactly as the S3 client would serve it.
+func dbFixture(t *testing.T) *dbFx {
+ t.Helper()
+ fx, member := instanceFixture(t, "code-db")
+ root := t.TempDir()
+ fx.srv.cfg.HibernateBucket = "data-test"
+ fx.srv.cfg.HibernatePrefix = "instances"
+ opens := new(atomic.Int32)
+ fx.srv.dbReplica = func(_ context.Context, prefix string) (replicaReader, error) {
+ opens.Add(1)
+ return lsfile.NewReplicaClient(filepath.Join(root, prefix)), nil
+ }
+ putDBRecord(t, fx, provision.Record{Host: dbTestHost})
+ return &dbFx{consoleFixture: fx, member: member, root: root, opens: opens}
+}
+
+// putDBRecord writes an instance record for titogo with the fields a
+// hibernating exec instance has, overridden by whatever rec sets.
+func putDBRecord(t *testing.T, fx *consoleFixture, rec provision.Record) {
+ t.Helper()
+ rec.App, rec.Account = "titogo", release.OpsAccount
+ if rec.Hibernate == nil {
+ yes := true
+ rec.Hibernate = &yes
+ }
+ body, err := json.Marshal(rec)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if _, err := fx.rel.Put(context.Background(), provision.RecordKey(rec.Account, rec.App, rec.Host), body, ""); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func (fx *dbFx) replicate(t *testing.T, host string, batches int) string {
+ t.Helper()
+ return dbdoortest.Replicate(t, filepath.Join(fx.root, "instances", host), batches)
+}
+
+func (fx *dbFx) url(p string) string { return fx.ts.URL + p }
+
+// A level the activator never writes is EMPTY, not missing. A 404 here
+// stops CalcRestorePlan at the first gap and the restore fails.
+func TestAPIDBListAnswersEmptyForUnwrittenLevels(t *testing.T) {
+ fx := dbFixture(t)
+ for level := 0; level <= dbdoor.MaxLevel; level++ {
+ status, out := getRaw(t, fx.url(dbdoor.ListPath("bab", "titogo", dbTestHost, level)), fx.member)
+ if status != http.StatusOK {
+ t.Fatalf("level %d = %d %s, want 200 with an empty list", level, status, out)
+ }
+ var res dbdoor.ListResponse
+ if err := json.Unmarshal([]byte(out), &res); err != nil || res.Files == nil || len(res.Files) != 0 {
+ t.Fatalf("level %d = %s, want {\"files\":[]}", level, out)
+ }
+ }
+}
+
+// Wire specifics 2 and 4: ascending by min_txid, and size, min_txid and
+// max_txid on every entry.
+func TestAPIDBListAscendsAndCarriesTheMandatoryFields(t *testing.T) {
+ fx := dbFixture(t)
+ fx.replicate(t, dbTestHost, 3)
+ status, out := getRaw(t, fx.url(dbdoor.ListPath("bab", "titogo", dbTestHost, 0)), fx.member)
+ if status != http.StatusOK {
+ t.Fatalf("list = %d %s", status, out)
+ }
+ var res dbdoor.ListResponse
+ if err := json.Unmarshal([]byte(out), &res); err != nil {
+ t.Fatal(err)
+ }
+ if len(res.Files) < 2 {
+ t.Fatalf("list = %s, want several L0 files", out)
+ }
+ for i, f := range res.Files {
+ if f.Size <= 0 || f.MinTXID == 0 || f.MaxTXID < f.MinTXID || f.Level != 0 {
+ t.Errorf("entry %d = %+v, want level 0 with size, min_txid and max_txid", i, f)
+ }
+ if i > 0 && f.MinTXID <= res.Files[i-1].MinTXID {
+ t.Errorf("entries %d and %d are not ascending by min_txid: %s", i-1, i, out)
+ }
+ }
+ for _, field := range []string{`"size"`, `"min_txid"`, `"max_txid"`} {
+ if !strings.Contains(out, field) {
+ t.Errorf("list body lacks %s: %s", field, out)
+ }
+ }
+}
+
+// get streams the file by coordinate, byte-for-byte, and honours the two
+// Range forms litestream's client sends.
+func TestAPIDBGetServesTheFileAndHonoursRange(t *testing.T) {
+ fx := dbFixture(t)
+ fx.replicate(t, dbTestHost, 1)
+ c := lsfile.NewReplicaClient(filepath.Join(fx.root, "instances", dbTestHost))
+ files := listFiles(t, c, 0)
+ f := files[0]
+ want, err := os.ReadFile(c.LTXFilePath(0, f.MinTXID, f.MaxTXID))
+ if err != nil {
+ t.Fatal(err)
+ }
+ u := fx.url(dbdoor.GetPath("bab", "titogo", dbTestHost, 0, uint64(f.MinTXID), uint64(f.MaxTXID)))
+ status, _, out := rawReq(t, http.MethodGet, u, fx.member, "")
+ if status != http.StatusOK || out != string(want) {
+ t.Fatalf("get = %d, %d bytes; want 200 and the file's %d bytes", status, len(out), len(want))
+ }
+ for _, rc := range []struct{ offset, size int64 }{{10, 0}, {10, 20}} {
+ req, _ := http.NewRequest(http.MethodGet, u, nil)
+ req.Header.Set("Authorization", "Bearer "+fx.member)
+ req.Header.Set("Range", dbdoor.RangeHeader(rc.offset, rc.size))
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ got, _ := io.ReadAll(res.Body)
+ res.Body.Close()
+ end := int64(len(want))
+ if rc.size > 0 {
+ end = rc.offset + rc.size
+ }
+ if res.StatusCode != http.StatusPartialContent || string(got) != string(want[rc.offset:end]) {
+ t.Errorf("range %+v = %d, %d bytes; want 206 and bytes %d-%d", rc, res.StatusCode, len(got), rc.offset, end)
+ }
+ }
+}
+
+func listFiles(t *testing.T, c *lsfile.ReplicaClient, level int) []*ltx.FileInfo {
+ t.Helper()
+ itr, err := c.LTXFiles(context.Background(), level, 0, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer itr.Close()
+ var out []*ltx.FileInfo
+ for itr.Next() {
+ out = append(out, itr.Item())
+ }
+ if len(out) == 0 {
+ t.Fatal("no files at that level")
+ }
+ return out
+}
+
+// A planned file that has gone is a 404 the client maps to os.ErrNotExist
+// (wire specific 5), which is what makes the whole-Restore retry work.
+func TestAPIDBGetOfAMissingFileIsA404(t *testing.T) {
+ fx := dbFixture(t)
+ fx.replicate(t, dbTestHost, 1)
+ status, _ := getRaw(t, fx.url(dbdoor.GetPath("bab", "titogo", dbTestHost, 0, 900, 901)), fx.member)
+ if status != http.StatusNotFound {
+ t.Fatalf("get of a missing coordinate = %d, want 404", status)
+ }
+}
+
+// Coordinate addressing exists to make lock.json unreachable. This is the
+// test that keeps the door from decaying into an object proxy.
+func TestAPIDBServesNoKeyOutsideItsCoordinates(t *testing.T) {
+ fx := dbFixture(t)
+ dir := filepath.Join(fx.root, "instances", dbTestHost)
+ fx.replicate(t, dbTestHost, 1)
+ if err := os.WriteFile(filepath.Join(dir, "lock.json"), []byte(`{"owner":"box-secret"}`), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ base := "/api/cli/apps/bab/titogo/db/" + dbTestHost
+ for _, bad := range []string{
+ base + "/ltx/0/../../lock.json",
+ base + "/lock.json",
+ base + "/ltx/lock.json",
+ base + "/ltx/0/lock.json",
+ base + "/ltx/10",
+ base + "/ltx/-1",
+ base + "/ltx/0/1",
+ base + "/ltx/0/0-1",
+ base + "/ltx/0/2-1",
+ base + "/ltx/10/1-2",
+ "/api/cli/apps/bab/titogo/db/..%2F..%2Fetc/ltx/0",
+ "/api/cli/apps/bab/titogo/db/" + "b.go.tito.io" + "/ltx/0",
+ } {
+ status, _, out := rawReq(t, http.MethodGet, fx.url(bad), fx.member, "")
+ if status == http.StatusOK || status == http.StatusPartialContent || strings.Contains(out, "box-secret") {
+ t.Errorf("%s = %d %s, want a refusal", bad, status, out)
+ }
+ }
+}
+
+// Not a member, no such app, no such instance: the same bytes http.NotFound
+// writes, matching instanceGate, so the door is no oracle for any of them.
+func TestAPIDBStrangersMeetTheByteIdentical404(t *testing.T) {
+ fx := dbFixture(t)
+ want := canonicalNotFound(t)
+ stranger := cliToken(t, fx.consoleFixture, "mallory@keymail.dev", "code-db-stranger")
+ for name, c := range map[string]struct{ tok, path string }{
+ "not a member": {stranger, dbdoor.ListPath("bab", "titogo", dbTestHost, 0)},
+ "no such app": {fx.member, dbdoor.ListPath("bab", "nosuch", dbTestHost, 0)},
+ "no such instance": {fx.member, dbdoor.ListPath("bab", "titogo", "nobody.go.tito.io", 0)},
+ "no such account": {fx.member, dbdoor.ListPath("zzzz", "titogo", dbTestHost, 0)},
+ "get, not a member": {stranger, dbdoor.GetPath("bab", "titogo", dbTestHost, 0, 1, 1)},
+ } {
+ status, h, body := rawReq(t, http.MethodGet, fx.url(c.path), c.tok, "")
+ assertNotFound(t, name, want, status, h, body)
+ }
+ if n := fx.opens.Load(); n != 0 {
+ t.Fatalf("the door reached for a replica %d times for callers it refuses", n)
+ }
+}
+
+// Every service principal, every role including admin, and the legacy app
+// token meet the closed-door refusal before any handler runs: the same bytes
+// whether or not the app and the instance exist, so it says nothing about
+// THIS app (Ruling, Task 4: the repo's zero-Set closed door, not a 404).
+func TestAPIDBRefusesEveryServicePrincipalTheSameWay(t *testing.T) {
+ fx := dbFixture(t)
+ fx.replicate(t, dbTestHost, 1)
+ appTok, err := fx.st.NewAppToken(OpsAccountID, "titogo", "alice@keymail.dev", "signup", time.Now())
+ if err != nil {
+ t.Fatal(err)
+ }
+ toks := map[string]string{"legacy app token": appTok}
+ for _, role := range []svcrole.Role{svcrole.Publish, svcrole.Instance, svcrole.Operate, svcrole.Admin} {
+ toks[string(role)] = serviceToken(t, fx.st, OpsAccountID, "svc-"+string(role), "titogo", role)
+ }
+ var first string
+ for name, tok := range toks {
+ for _, p := range []string{
+ dbdoor.ListPath("bab", "titogo", dbTestHost, 0),
+ dbdoor.GetPath("bab", "titogo", dbTestHost, 0, 1, 1),
+ dbdoor.ListPath("bab", "nosuch", "nobody.go.tito.io", 0),
+ } {
+ status, _, body := rawReq(t, http.MethodGet, fx.url(p), tok, "")
+ if status != http.StatusForbidden || !strings.Contains(body, "closed to service credentials") {
+ t.Errorf("%s at %s = %d %s, want the closed-door refusal", name, p, status, body)
+ }
+ if first == "" {
+ first = body
+ } else if body != first {
+ t.Errorf("%s at %s: refusal varies (%q vs %q)", name, p, body, first)
+ }
+ }
+ }
+ if n := fx.opens.Load(); n != 0 {
+ t.Fatalf("the door reached for a replica %d times for service principals", n)
+ }
+}
+
+// The residency and pooled refusals are answered with no replica client
+// ever built: a 409 naming the cause, before the bucket.
+func TestAPIDBResidencyRefusalsNeverReachTheBucket(t *testing.T) {
+ fx := dbFixture(t)
+ fx.srv.cfg.Region = "ie"
+ putDBRecord(t, fx.consoleFixture, provision.Record{Host: "uk.go.tito.io", Region: "uk"})
+ putDBRecord(t, fx.consoleFixture, provision.Record{Host: "fleet.go.tito.io", Region: "fleet/acme"})
+ putDBRecord(t, fx.consoleFixture, provision.Record{Host: "pool.go.tito.io", Region: "ie", Pool: "tito"})
+ for host, want := range map[string]string{
+ "uk.go.tito.io": "uk",
+ "fleet.go.tito.io": "fleet",
+ "pool.go.tito.io": "pool",
+ dbTestHost: "",
+ } {
+ status, out := getRaw(t, fx.url(dbdoor.ListPath("bab", "titogo", host, 0)), fx.member)
+ if status != http.StatusConflict || !strings.Contains(out, want) {
+ t.Errorf("%s = %d %s, want a 409 naming %q", host, status, out, want)
+ }
+ }
+ if n := fx.opens.Load(); n != 0 {
+ t.Fatalf("the door reached for a replica %d times before refusing on residency", n)
+ }
+}
+
+// Only an activator-managed exec instance has a replica the platform
+// writes. Everything else is refused by class, before the bucket.
+func TestAPIDBRefusesRoutesTheActivatorDoesNotReplicate(t *testing.T) {
+ fx := dbFixture(t)
+ no := false
+ putDBRecord(t, fx.consoleFixture, provision.Record{Host: "svc.go.tito.io", Kind: "service"})
+ putDBRecord(t, fx.consoleFixture, provision.Record{Host: "addr.go.tito.io", Addr: "10.0.0.1:8080"})
+ putDBRecord(t, fx.consoleFixture, provision.Record{Host: "awake.go.tito.io", Hibernate: &no})
+ for _, host := range []string{"svc.go.tito.io", "addr.go.tito.io", "awake.go.tito.io"} {
+ status, out := getRaw(t, fx.url(dbdoor.ListPath("bab", "titogo", host, 0)), fx.member)
+ if status != http.StatusConflict {
+ t.Errorf("%s = %d %s, want a 409 refusal", host, status, out)
+ }
+ }
+ if n := fx.opens.Load(); n != 0 {
+ t.Fatalf("the door reached for a replica %d times for routes with none", n)
+ }
+}
+
+// No data bucket configured is a 503, never a door that fails open.
+func TestAPIDBWithoutADataBucketIs503(t *testing.T) {
+ fx := dbFixture(t)
+ fx.srv.cfg.HibernateBucket = ""
+ status, out := getRaw(t, fx.url(dbdoor.ListPath("bab", "titogo", dbTestHost, 0)), fx.member)
+ if status != http.StatusServiceUnavailable {
+ t.Fatalf("no bucket = %d %s, want 503", status, out)
+ }
+ if fx.opens.Load() != 0 {
+ t.Fatal("the door reached for a replica with no bucket configured")
+ }
+}
+
+// S3 refusing the read is the residency refusal as a backstop (spec §6),
+// never a generic 502 — and the store's own error text, which carries the
+// bucket name, never reaches the caller.
+func TestAPIDBAccessDeniedIsTheResidencyRefusal(t *testing.T) {
+ fx := dbFixture(t)
+ fx.srv.dbReplica = func(context.Context, string) (replicaReader, error) {
+ return deniedReplica{}, nil
+ }
+ for _, p := range []string{
+ dbdoor.ListPath("bab", "titogo", dbTestHost, 0),
+ dbdoor.GetPath("bab", "titogo", dbTestHost, 0, 1, 1),
+ } {
+ status, out := getRaw(t, fx.url(p), fx.member)
+ if status != http.StatusConflict || strings.Contains(out, "data-test") || strings.Contains(out, "secret-bucket") {
+ t.Errorf("%s = %d %s, want the residency 409 with no bucket name", p, status, out)
+ }
+ }
+}
+
+type deniedReplica struct{}
+
+func (deniedReplica) LTXFiles(context.Context, int, ltx.TXID, bool) (ltx.FileIterator, error) {
+ return nil, &smithy.GenericAPIError{Code: "AccessDenied", Message: "no access to secret-bucket"}
+}
+
+func (deniedReplica) OpenLTXFile(context.Context, int, ltx.TXID, ltx.TXID, int64, int64) (io.ReadCloser, error) {
+ return nil, &smithy.GenericAPIError{Code: "AccessDenied", Message: "no access to secret-bucket"}
+}
+
+// Every successful open of a file writes an audit row naming the instance
+// and the TXID span served (spec §6 "Who may open it").
+func TestAPIDBGetIsAudited(t *testing.T) {
+ fx := dbFixture(t)
+ fx.replicate(t, dbTestHost, 1)
+ f := listFiles(t, lsfile.NewReplicaClient(filepath.Join(fx.root, "instances", dbTestHost)), 0)[0]
+ if status, _ := getRaw(t, fx.url(dbdoor.GetPath("bab", "titogo", dbTestHost, 0, uint64(f.MinTXID), uint64(f.MaxTXID))), fx.member); status != http.StatusOK {
+ t.Fatalf("get = %d", status)
+ }
+ evs, err := fx.st.Events(OpsAccountID, "titogo", 10)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, e := range evs {
+ if e.Action == "db-read" && e.Target == dbTestHost && e.Address == "alice@keymail.dev" &&
+ strings.Contains(e.Detail, fmt.Sprint(uint64(f.MaxTXID))) {
+ return
+ }
+ }
+ t.Fatalf("no db-read audit row naming %s and txid %d: %+v", dbTestHost, f.MaxTXID, evs)
+}
+
+// One replica client per prefix: a restore's dozens of requests share one
+// transport rather than paying a handshake each. And the production reader
+// carries the configured bucket and the resolved prefix, nothing a request
+// supplied.
+func TestCachedReadReplicaIsOnePerPrefix(t *testing.T) {
+ t.Setenv("AWS_EC2_METADATA_DISABLED", "true")
+ s := &Server{}
+ s.cfg.HibernateBucket = "data-ie"
+ s.cfg.HibernateRegion = "eu-west-1"
+ ctx := context.Background()
+ a1, err := s.cachedReadReplica(ctx, "instances/a.example.com")
+ if err != nil {
+ t.Fatal(err)
+ }
+ a2, _ := s.cachedReadReplica(ctx, "instances/a.example.com")
+ b, _ := s.cachedReadReplica(ctx, "instances/b.example.com")
+ if a1 != a2 {
+ t.Error("the same prefix built two clients")
+ }
+ if a1 == b {
+ t.Error("two prefixes shared one client")
+ }
+ c, ok := a1.(*lss3.ReplicaClient)
+ if !ok || c.Bucket != "data-ie" || c.Path != "instances/a.example.com" || c.Region != "eu-west-1" {
+ t.Fatalf("client = %#v, want the configured bucket, region and the resolved prefix", a1)
+ }
+}
+
+// The door has its own token-keyed budget, separate from the instance
+// verbs': spending it does not touch the instance budget, and it applies
+// only after membership, so a stranger cannot learn anything from a 429.
+func TestAPIDBHasItsOwnBudget(t *testing.T) {
+ fx := dbFixture(t)
+ fx.srv.cli.db = newRateLimiter(2, cliRateWindow)
+ u := fx.url(dbdoor.ListPath("bab", "titogo", dbTestHost, 0))
+ for i := 0; i < 2; i++ {
+ if status, out := getRaw(t, u, fx.member); status != http.StatusOK {
+ t.Fatalf("request %d = %d %s", i, status, out)
+ }
+ }
+ if status, _ := getRaw(t, u, fx.member); status != http.StatusTooManyRequests {
+ t.Fatalf("third request = %d, want 429", status)
+ }
+ if !fx.srv.cli.instances.allow(hashToken(fx.member), time.Now()) {
+ t.Fatal("the db door spent the instance verbs' budget")
+ }
+ stranger := cliToken(t, fx.consoleFixture, "mallory@keymail.dev", "code-db-budget")
+ for i := 0; i < 3; i++ {
+ if status, _ := getRaw(t, u, stranger); status != http.StatusNotFound {
+ t.Fatalf("stranger request %d = %d, want the 404 before any budget", i, status)
+ }
+ }
+}
+
+// End to end: the CLI's read-only client restores a real replica through
+// the real door, with a member's token, and gets every row.
+func TestAPIDBRestoresThroughTheDoor(t *testing.T) {
+ fx := dbFixture(t)
+ fx.replicate(t, dbTestHost, 3)
+ // The tightest bound there is, and a restore still completes under it.
+ // (Small files finish server-side before the client reads them, so the
+ // one-stream-at-a-time property itself is pinned in internal/dbdoor.)
+ fx.srv.dbStreams = newDBStreamLimits(1, 1)
+ do := func(ctx context.Context, tail string, h http.Header) (*http.Response, error) {
+ req, err := http.NewRequestWithContext(ctx, http.MethodGet, fx.url(dbdoor.Base("bab", "titogo", dbTestHost)+tail), nil)
+ if err != nil {
+ return nil, err
+ }
+ req.Header = h.Clone()
+ req.Header.Set("Authorization", "Bearer "+fx.member)
+ return http.DefaultClient.Do(req)
+ }
+ got, err := dbdoor.RestoreTo(context.Background(), dbdoor.NewClient(do), t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(got.Dir)
+ db, err := sql.Open("sqlite", "file:"+got.Path+"?mode=ro")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ var n int
+ if err := db.QueryRow(`SELECT count(*) FROM t`).Scan(&n); err != nil {
+ t.Fatal(err)
+ }
+ if n != 3*dbdoortest.Rows || got.TXID == 0 || got.At.IsZero() {
+ t.Fatalf("restored %d rows, txid %d at %v; want %d rows and an as-of", n, got.TXID, got.At, 3*dbdoortest.Rows)
+ }
+}
+
+// A ranged GET is still a read of the file. Listing the coordinates and
+// fetching each one with Range: bytes=1- returns all but one byte of every
+// file, so every successful GET writes a row, and a resumed one says where
+// it started.
+func TestAPIDBRangedGetIsAudited(t *testing.T) {
+ fx := dbFixture(t)
+ fx.replicate(t, dbTestHost, 1)
+ f := listFiles(t, lsfile.NewReplicaClient(filepath.Join(fx.root, "instances", dbTestHost)), 0)[0]
+ req, _ := http.NewRequest(http.MethodGet, fx.url(dbdoor.GetPath("bab", "titogo", dbTestHost, 0, uint64(f.MinTXID), uint64(f.MaxTXID))), nil)
+ req.Header.Set("Authorization", "Bearer "+fx.member)
+ req.Header.Set("Range", "bytes=1-")
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ io.Copy(io.Discard, res.Body)
+ res.Body.Close()
+ if res.StatusCode != http.StatusPartialContent {
+ t.Fatalf("ranged get = %d", res.StatusCode)
+ }
+ evs, err := fx.st.Events(OpsAccountID, "titogo", 10)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, e := range evs {
+ if e.Action == "db-read" && e.Target == dbTestHost && strings.Contains(e.Detail, "byte 1") {
+ return
+ }
+ }
+ t.Fatalf("a ranged read wrote no db-read row naming its start: %+v", evs)
+}
+
+// stallReplica serves listings from a file replica but opens files whose
+// reads never finish: each blocks until the request's context ends, the way
+// a stalled S3 read does, or produces zeros forever when endless is set.
+type stallReplica struct {
+ replicaReader
+ endless bool
+ opened chan struct{}
+}
+
+func (s *stallReplica) OpenLTXFile(ctx context.Context, level int, min, max ltx.TXID, offset, size int64) (io.ReadCloser, error) {
+ select {
+ case s.opened <- struct{}{}:
+ default:
+ }
+ if s.endless {
+ return io.NopCloser(zeros{}), nil
+ }
+ return io.NopCloser(ctxReader{ctx}), nil
+}
+
+type ctxReader struct{ ctx context.Context }
+
+func (r ctxReader) Read([]byte) (int, error) { <-r.ctx.Done(); return 0, r.ctx.Err() }
+
+type zeros struct{}
+
+func (zeros) Read(p []byte) (int, error) { clear(p); return len(p), nil }
+
+func (fx *dbFx) stall(endless bool) *stallReplica {
+ sr := &stallReplica{endless: endless, opened: make(chan struct{}, 16)}
+ fx.srv.dbReplica = func(_ context.Context, prefix string) (replicaReader, error) {
+ sr.replicaReader = lsfile.NewReplicaClient(filepath.Join(fx.root, prefix))
+ return sr, nil
+ }
+ return sr
+}
+
+func (fx *dbFx) memberToken(t *testing.T, addr, code string) string {
+ t.Helper()
+ if err := fx.st.AddMember(OpsAccountID, addr, RoleMember, time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ return cliToken(t, fx.consoleFixture, addr, code)
+}
+
+// startGet begins a GET and returns once the door has opened the file, so
+// the stream holds its slot; cancel ends it.
+func (fx *dbFx) startGet(t *testing.T, sr *stallReplica, tok string) (stop func()) {
+ t.Helper()
+ ctx, cancel := context.WithCancel(context.Background())
+ req, _ := http.NewRequestWithContext(ctx, http.MethodGet, fx.url(dbdoor.GetPath("bab", "titogo", dbTestHost, 0, 1, 1)), nil)
+ req.Header.Set("Authorization", "Bearer "+tok)
+ done := make(chan struct{})
+ go func() {
+ defer close(done)
+ if res, err := http.DefaultClient.Do(req); err == nil {
+ io.Copy(io.Discard, res.Body)
+ res.Body.Close()
+ }
+ }()
+ select {
+ case <-sr.opened:
+ case <-time.After(5 * time.Second):
+ t.Fatal("the stream never opened")
+ }
+ return func() { cancel(); <-done }
+}
+
+func (fx *dbFx) activeStreams() int {
+ fx.srv.dbStreams.mu.Lock()
+ defer fx.srv.dbStreams.mu.Unlock()
+ return fx.srv.dbStreams.total
+}
+
+func (fx *dbFx) waitIdle(t *testing.T, within time.Duration) {
+ t.Helper()
+ deadline := time.Now().Add(within)
+ for fx.activeStreams() != 0 {
+ if time.Now().After(deadline) {
+ t.Fatalf("%d streams still hold slots after %v", fx.activeStreams(), within)
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+}
+
+// Slow downloads must not accumulate: active file streams are bounded per
+// principal and across the console, and a refusal says when to come back.
+func TestAPIDBBoundsConcurrentStreams(t *testing.T) {
+ fx := dbFixture(t)
+ fx.replicate(t, dbTestHost, 1)
+ fx.srv.dbStreams = newDBStreamLimits(2, 1)
+ sr := fx.stall(false)
+ bob := fx.memberToken(t, "bob@keymail.dev", "code-db-bob")
+ carol := fx.memberToken(t, "carol@keymail.dev", "code-db-carol")
+ u := fx.url(dbdoor.GetPath("bab", "titogo", dbTestHost, 0, 1, 1))
+
+ stopAlice := fx.startGet(t, sr, fx.member)
+ status, h, body := rawReq(t, http.MethodGet, u, fx.member, "")
+ if status != http.StatusTooManyRequests || h.Get("Retry-After") == "" {
+ t.Fatalf("second stream for one person = %d %q %s, want 429 with Retry-After", status, h.Get("Retry-After"), body)
+ }
+ stopBob := fx.startGet(t, sr, bob)
+ status, h, body = rawReq(t, http.MethodGet, u, carol, "")
+ if status != http.StatusTooManyRequests || h.Get("Retry-After") == "" {
+ t.Fatalf("stream past the console's bound = %d %q %s, want 429 with Retry-After", status, h.Get("Retry-After"), body)
+ }
+ stopAlice()
+ stopBob()
+ fx.waitIdle(t, 5*time.Second)
+ stopAgain := fx.startGet(t, sr, fx.member)
+ stopAgain()
+}
+
+// A stalled bucket read ends after the idle deadline and frees its slot.
+func TestAPIDBStalledReadIsCut(t *testing.T) {
+ fx := dbFixture(t)
+ fx.replicate(t, dbTestHost, 1)
+ fx.srv.dbStreamIdle = 200 * time.Millisecond
+ sr := fx.stall(false)
+ start := time.Now()
+ status, _, _ := rawReq(t, http.MethodGet, fx.url(dbdoor.GetPath("bab", "titogo", dbTestHost, 0, 1, 1)), fx.member, "")
+ <-sr.opened
+ if time.Since(start) > 5*time.Second {
+ t.Fatalf("a stalled read held the request for %v", time.Since(start))
+ }
+ _ = status
+ fx.waitIdle(t, 5*time.Second)
+}
+
+// A client that stops reading is cut by the renewable write deadline, so a
+// trickling or abandoned download cannot hold a slot forever.
+func TestAPIDBStalledClientIsCut(t *testing.T) {
+ fx := dbFixture(t)
+ fx.replicate(t, dbTestHost, 1)
+ fx.srv.dbStreamIdle = 300 * time.Millisecond
+ sr := fx.stall(true)
+ req, _ := http.NewRequest(http.MethodGet, fx.url(dbdoor.GetPath("bab", "titogo", dbTestHost, 0, 1, 1)), nil)
+ req.Header.Set("Authorization", "Bearer "+fx.member)
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer res.Body.Close() // never read: the client has stalled
+ <-sr.opened
+ fx.waitIdle(t, 15*time.Second)
+}
+
+// recConn records, in order, every write and every write-deadline change on
+// a server-side connection.
+type recConn struct {
+ net.Conn
+ mu *sync.Mutex
+ events *[]string
+}
+
+func (c recConn) Write(p []byte) (int, error) {
+ c.mu.Lock()
+ *c.events = append(*c.events, "write")
+ c.mu.Unlock()
+ return c.Conn.Write(p)
+}
+
+func (c recConn) SetWriteDeadline(t time.Time) error {
+ c.mu.Lock()
+ if t.IsZero() {
+ *c.events = append(*c.events, "clear")
+ } else {
+ *c.events = append(*c.events, "deadline")
+ }
+ c.mu.Unlock()
+ return c.Conn.SetWriteDeadline(t)
+}
+
+type recListener struct {
+ net.Listener
+ mu sync.Mutex
+ events []string
+}
+
+func (l *recListener) Accept() (net.Conn, error) {
+ c, err := l.Listener.Accept()
+ if err != nil {
+ return nil, err
+ }
+ return recConn{Conn: c, mu: &l.mu, events: &l.events}, nil
+}
+
+// net/http flushes what is still buffered (the last bytes, the chunked
+// terminator) in finishRequest, AFTER the handler returns, and clears the
+// write deadline itself only after that. A handler that clears it on the
+// way out leaves that final flush with no deadline, so a client that stops
+// reading near the end holds the connection forever, outside the stream
+// bound. No byte may ever be written with the deadline cleared.
+func TestAPIDBNeverWritesWithoutADeadline(t *testing.T) {
+ fx := dbFixture(t)
+ fx.replicate(t, dbTestHost, 1)
+ f := listFiles(t, lsfile.NewReplicaClient(filepath.Join(fx.root, "instances", dbTestHost)), 0)[0]
+ ts := httptest.NewUnstartedServer(fx.srv)
+ rl := &recListener{Listener: ts.Listener}
+ ts.Listener = rl
+ ts.Start()
+ defer ts.Close()
+ req, _ := http.NewRequest(http.MethodGet, ts.URL+dbdoor.GetPath("bab", "titogo", dbTestHost, 0, uint64(f.MinTXID), uint64(f.MaxTXID)), nil)
+ req.Header.Set("Authorization", "Bearer "+fx.member)
+ req.Close = true
+ res, err := http.DefaultClient.Do(req)
+ if err != nil {
+ t.Fatal(err)
+ }
+ io.Copy(io.Discard, res.Body)
+ res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ t.Fatalf("get = %d", res.StatusCode)
+ }
+ deadline := time.Now().Add(5 * time.Second)
+ for {
+ rl.mu.Lock()
+ events := append([]string(nil), rl.events...)
+ rl.mu.Unlock()
+ if len(events) > 0 && events[len(events)-1] == "clear" || time.Now().After(deadline) {
+ armed := false
+ for i, e := range events {
+ switch e {
+ case "deadline":
+ armed = true
+ case "clear":
+ armed = false
+ case "write":
+ if !armed {
+ t.Fatalf("write %d happened with no write deadline: %v", i, events)
+ }
+ }
+ }
+ if !armed && events[len(events)-1] != "clear" {
+ t.Fatalf("events = %v", events)
+ }
+ return
+ }
+ time.Sleep(10 * time.Millisecond)
+ }
+}
diff --git a/internal/console/authcli.go b/internal/console/authcli.go
index 9fdd2e11c..7d09d555c 100644
--- a/internal/console/authcli.go
+++ b/internal/console/authcli.go
@@ -322,6 +322,10 @@ func retryAfter(w http.ResponseWriter, d time.Duration) {
// Server so a test gets a fresh set per fixture.
type cliLimits struct {
start, poll, pollIP, guessAddr, guessIP, write, ship, instances, create *rateLimiter
+ // db is the `carlos db` door's own budget, keyed by token (cliDBBurst,
+ // apidb.go): a restore is dozens of reads, a different shape from the
+ // instance verbs.
+ db *rateLimiter
// svcCreate, svcWrite and svcShip are the SERVICE side of create/write/
// ship — separate counters, separate keys (principalKey's NAME, not the
// person's ADDRESS or the ship-person's token hash), so a credential's
@@ -361,6 +365,7 @@ func newCLILimits() *cliLimits {
// most of this work is not a person, so the address-keyed write
// budget is the wrong principal for it in both directions.
instances: newRateLimiter(cliInstanceBurst, cliRateWindow),
+ db: newRateLimiter(cliDBBurst, cliRateWindow),
// Create gets its own budget on its own (day-long) window, keyed by
// address — the stolen-token ceiling handleAPICreateApp explains.
create: newRateLimiter(cliCreateBurst, cliCreateWindow),
@@ -627,6 +632,14 @@ func (s *Server) registerCLIRoutes() {
s.cliRoute("POST /api/cli/apps/{account}/{app}/instances/{host}/move/cut", svcrole.Allow(svcrole.Instance), s.handleAPIInstanceMoveCut)
s.cliRoute("DELETE /api/cli/apps/{account}/{app}/instances/{host}/move", svcrole.Allow(svcrole.Instance), s.handleAPIInstanceMoveAbort)
s.cliRoute("DELETE /api/cli/apps/{account}/{app}/instances/{host}", svcrole.Allow(svcrole.Instance), s.handleAPIInstanceDelete)
+ // The `carlos db` replica door (apidb.go; spec 2026-09-02 §6): an
+ // instance's replicated LTX files, by coordinate, for a member to
+ // restore on their own machine. The zero Set, deliberately: the payload
+ // is the whole customer database, and no legitimate credential needs it
+ // (an app reads its own database live, and CI has no business pulling
+ // production data). admin is refused with the rest.
+ s.cliRoute("GET /api/cli/apps/{account}/{app}/db/{host}/ltx/{level}", svcrole.Set{}, s.handleAPIDBList)
+ s.cliRoute("GET /api/cli/apps/{account}/{app}/db/{host}/ltx/{level}/{span}", svcrole.Set{}, s.handleAPIDBGet)
// The zero Set, deliberately: a credential may never mint a credential,
// including itself. This is the escalation cut (spec §3). It takes no
// authority away — handleAPIInstanceToken gates on apiAccountApp, which
diff --git a/internal/console/server.go b/internal/console/server.go
index 80838075e..e1902a10e 100644
--- a/internal/console/server.go
+++ b/internal/console/server.go
@@ -482,6 +482,21 @@ type Config struct {
// and nothing about placement changes when it is set. "" is legal: the
// regions endpoint then reports no home row and an empty default.
HomeRegion string
+ // HibernateBucket/Region/Prefix name the region data bucket the
+ // `carlos db` door reads (apidb.go; spec 2026-09-02 §6, Option A): the
+ // same CARLOS_HIBERNATE_* triple the activator on this box writes
+ // instances/<host>/ with, read through activator.ConfigFromEnv so the
+ // two cannot parse it differently. Read-only, in-region, LTX files by
+ // coordinate only. An empty bucket disables the door (503) rather than
+ // failing it open.
+ HibernateBucket string
+ HibernateRegion string
+ HibernatePrefix string
+ // Region is this console's own placement (CARLOS_REGION, the value the
+ // box agent beside it reconciles with), compared against an instance
+ // record's Region before any S3 call. Empty on the home region, whose
+ // records keep region "" too (see HomeRegion).
+ Region string
ReleasePub ed25519.PublicKey // pinned channel-pointer verification key
// Key signs channel pointers for promote/rollback (CARLOS_RELEASE_KEY).
// nil is a legal deployment: the console runs read-only, the promotion
@@ -709,6 +724,15 @@ type Server struct {
instances *instanceFetcher
secure bool // Secure cookies iff the origin is https
cli *cliLimits // rate limits on the CLI-identity endpoints (authcli.go)
+ // dbReplica opens the read side of one instance's replica for the
+ // `carlos db` door (apidb.go). NewServer points it at a small cache of
+ // lease.ReadReplicaClient values; tests point it at a file replica.
+ dbReplica func(ctx context.Context, prefix string) (replicaReader, error)
+ dbClients dbReplicaCache
+ // dbStreams bounds the db door's active file streams, and dbStreamIdle
+ // is how long one may go without moving a byte (apidb.go).
+ dbStreams *dbStreamLimits
+ dbStreamIdle time.Duration
// cliRoles is pattern -> allowed roles, recorded by cliRoute as it
// registers each /api/cli/ door (authcli.go). Written once, during
// registerCLIRoutes, before the server serves anything, and only read
@@ -1132,6 +1156,10 @@ func NewServer(cfg Config, st *Store) (*Server, error) {
s.mux.HandleFunc("/fleet/economics/tier", s.handleEconomicsTierSave)
// CLI identity: the device-code login `carlos auth login` drives, the
// approval page, and the tokens an owner can see and kill (authcli.go).
+ // The db door's production replica reader (apidb.go); tests repoint it.
+ s.dbReplica = s.cachedReadReplica
+ s.dbStreams = newDBStreamLimits(dbMaxStreams, dbMaxStreamsPerPrincipal)
+ s.dbStreamIdle = dbStreamIdleDefault
s.registerCLIRoutes()
// The email API's ten doors (apimail.go, tenant-email task 17) —
// registered separately from registerCLIRoutes because this task's file
diff --git a/internal/console/svcroutes_test.go b/internal/console/svcroutes_test.go
index ced377726..b966f2d64 100644
--- a/internal/console/svcroutes_test.go
+++ b/internal/console/svcroutes_test.go
@@ -168,6 +168,13 @@ var closedDoors = []string{
// whose rules it never met. Shut for the same reason the manifest door
// is — an app may not rewrite the bounds it runs inside.
"POST /api/cli/apps/{account}/{app}/instances/{host}/channel",
+ // The `carlos db` replica door (apidb.go) serves the whole customer
+ // database, file by file. No credential the app holds needs it: the app
+ // reads its own database live, and CI has no business pulling
+ // production data. Shut to every role, admin included (spec 2026-09-02
+ // §6, "Who may open it").
+ "GET /api/cli/apps/{account}/{app}/db/{host}/ltx/{level}",
+ "GET /api/cli/apps/{account}/{app}/db/{host}/ltx/{level}/{span}",
// The build probe is what `carlos deploy`'s watch TRUSTS in place of the
// X-Carlos-Version header — the header being the thing that reports a
// release before any process has restarted. A credential that could
@@ -783,6 +790,8 @@ func TestEveryClosedDoorRefusesAServiceCredential(t *testing.T) {
"{name}", "hub",
"{host}", "web.go.tito.io",
"{id}", "deadbeefdeadbeef",
+ "{level}", "0",
+ "{span}", "1-1",
).Replace(path)
if strings.Contains(u, "{") {
t.Fatalf("closedDoors entry %q has a wildcard this test does not fill", pattern)
@@ -800,8 +809,8 @@ func TestEveryClosedDoorRefusesAServiceCredential(t *testing.T) {
}
seen++
}
- if seen != len(closedDoors) || seen != 34 {
- t.Errorf("walked %d closed doors, want all %d (and 34 today: instance-token, three door-key doors, the routes door, instances-manifest, sidecars-manifest, the route-channel move, six fleets, four services, two email credentials, email pause, the build probe, the live tenancy read, five system doors, two deployment-wide antivirus, and four store-entitlement doors)",
+ if seen != len(closedDoors) || seen != 36 {
+ t.Errorf("walked %d closed doors, want all %d (and 36 today: instance-token, three door-key doors, the routes door, instances-manifest, sidecars-manifest, the route-channel move, two db replica doors, six fleets, four services, two email credentials, email pause, the build probe, the live tenancy read, five system doors, two deployment-wide antivirus, and four store-entitlement doors)",
seen, len(closedDoors))
}
}
diff --git a/internal/dbdoor/client.go b/internal/dbdoor/client.go
new file mode 100644
index 000000000..7632dceff
--- /dev/null
+++ b/internal/dbdoor/client.go
@@ -0,0 +1,157 @@
+package dbdoor
+
+import (
+ "context"
+ "encoding/json"
+ "errors"
+ "fmt"
+ "io"
+ "log/slog"
+ "net/http"
+ "os"
+ "strconv"
+ "strings"
+ "time"
+
+ "github.com/benbjohnson/litestream"
+ "github.com/superfly/ltx"
+)
+
+// Doer sends one GET to the door for one instance. tail is relative to the
+// instance's door root (ListTail or GetTail); the caller owns the console
+// URL, the bearer and the account segment, which is why the client never
+// sees them. The caller's response body is the client's to close.
+type Doer func(ctx context.Context, tail string, header http.Header) (*http.Response, error)
+
+// Client is the CLI's litestream.ReplicaClient over the db door (spec
+// 2026-09-02 §6, "The read-only replica client"), the way internal/fleetapi
+// implements release.Store over the edge store door: the member has no AWS
+// credentials, by design.
+//
+// It supplies the five methods a restore needs (Type, Init, LTXFiles,
+// OpenLTXFile, SetLogger). The three that mutate a replica return
+// errReadOnly and have no other body. It must never implement
+// litestream.ReplicaClientV3, or Restore would take the v0.3 path.
+type Client struct {
+ do Doer
+ logger *slog.Logger
+}
+
+// NewClient returns a read-only replica client that reads through do.
+func NewClient(do Doer) *Client {
+ return &Client{do: do, logger: slog.Default()}
+}
+
+// errReadOnly is not a guard that can be switched off: these three methods
+// have no other body. The two-replicators hazard internal/caps and
+// internal/lease both warn about is unreachable from this client by
+// construction, and a future edit that wanted to reach it would have to add
+// the capability first.
+var errReadOnly = errors.New("dbdoor: this client is read-only; it restores replicas and never writes them")
+
+// DoorError is a refusal from the door, carrying the console's own sentence
+// (its {"error"} body) so the CLI can show it as written.
+type DoorError struct {
+ Status int
+ Message string
+ // RetryAfter is the door's Retry-After, in whole seconds, when it sent
+ // one (a 429 from a full stream bound does); zero otherwise.
+ RetryAfter time.Duration
+}
+
+func (e *DoorError) Error() string {
+ if e.Message == "" {
+ return fmt.Sprintf("the console answered %d", e.Status)
+ }
+ return e.Message
+}
+
+func (c *Client) Type() string { return "carlos-db" }
+
+// Init is a no-op: the transport needs no setup, and litestream requires
+// it to be idempotent.
+func (c *Client) Init(context.Context) error { return nil }
+
+func (c *Client) SetLogger(l *slog.Logger) { c.logger = l }
+
+// LTXFiles lists one level. seek is filtered here (wire specific 3), with
+// litestream's own rule: files whose MinTXID is below seek are skipped.
+// useMetadata is ignored; nothing in a non-timestamp restore reads
+// CreatedAt except debug logs.
+func (c *Client) LTXFiles(ctx context.Context, level int, seek ltx.TXID, _ bool) (ltx.FileIterator, error) {
+ res, err := c.do(ctx, ListTail(level), http.Header{})
+ if err != nil {
+ return nil, err
+ }
+ defer res.Body.Close()
+ if res.StatusCode != http.StatusOK {
+ return nil, doorError(res)
+ }
+ var list ListResponse
+ if err := json.NewDecoder(io.LimitReader(res.Body, 16<<20)).Decode(&list); err != nil {
+ return nil, fmt.Errorf("dbdoor: unreadable listing for level %d: %w", level, err)
+ }
+ infos := make([]*ltx.FileInfo, 0, len(list.Files))
+ for _, f := range list.Files {
+ if ltx.TXID(f.MinTXID) < seek {
+ continue
+ }
+ infos = append(infos, &ltx.FileInfo{
+ Level: level, MinTXID: ltx.TXID(f.MinTXID), MaxTXID: ltx.TXID(f.MaxTXID),
+ Size: f.Size, CreatedAt: f.CreatedAt,
+ })
+ }
+ return ltx.NewFileInfoSliceIterator(infos), nil
+}
+
+// OpenLTXFile opens one file by coordinate, sending the S3 client's exact
+// Range form. A 404 is os.ErrNotExist (wire specific 5): that is what lets
+// RestoreTo tell a file that vanished under the plan from a refusal.
+func (c *Client) OpenLTXFile(ctx context.Context, level int, minTXID, maxTXID ltx.TXID, offset, size int64) (io.ReadCloser, error) {
+ h := http.Header{}
+ h.Set("Range", RangeHeader(offset, size))
+ res, err := c.do(ctx, GetTail(level, uint64(minTXID), uint64(maxTXID)), h)
+ if err != nil {
+ return nil, err
+ }
+ switch res.StatusCode {
+ case http.StatusOK, http.StatusPartialContent:
+ return res.Body, nil
+ case http.StatusNotFound:
+ res.Body.Close()
+ return nil, fmt.Errorf("dbdoor: level %d %s: %w", level, Span(uint64(minTXID), uint64(maxTXID)), os.ErrNotExist)
+ default:
+ defer res.Body.Close()
+ return nil, doorError(res)
+ }
+}
+
+func (c *Client) WriteLTXFile(context.Context, int, ltx.TXID, ltx.TXID, io.Reader) (*ltx.FileInfo, error) {
+ return nil, errReadOnly
+}
+
+func (c *Client) DeleteLTXFiles(context.Context, []*ltx.FileInfo) error { return errReadOnly }
+
+func (c *Client) DeleteAll(context.Context) error { return errReadOnly }
+
+// doorError reads a refusal's {"error"} sentence. A body that is not that
+// shape (the stdlib's 404 text, a proxy's page) keeps only the status.
+func doorError(res *http.Response) error {
+ raw, _ := io.ReadAll(io.LimitReader(res.Body, 64<<10))
+ var body struct {
+ Error string `json:"error"`
+ }
+ e := &DoorError{Status: res.StatusCode}
+ if json.Unmarshal(raw, &body) == nil {
+ e.Message = strings.TrimSpace(body.Error)
+ }
+ // Seconds only (the door never sends a date), capped so a hostile or
+ // broken value cannot park a download for hours.
+ if secs, err := strconv.Atoi(strings.TrimSpace(res.Header.Get("Retry-After"))); err == nil && secs > 0 {
+ // Clamp the count before multiplying: a large one would wrap.
+ e.RetryAfter = time.Duration(min(secs, 60)) * time.Second
+ }
+ return e
+}
+
+var _ litestream.ReplicaClient = (*Client)(nil)
diff --git a/internal/dbdoor/client_test.go b/internal/dbdoor/client_test.go
new file mode 100644
index 000000000..8b3952258
--- /dev/null
+++ b/internal/dbdoor/client_test.go
@@ -0,0 +1,312 @@
+package dbdoor
+
+import (
+ "context"
+ "database/sql"
+ "encoding/json"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "sync"
+ "testing"
+ "time"
+
+ "github.com/benbjohnson/litestream"
+ lsfile "github.com/benbjohnson/litestream/file"
+ "github.com/superfly/ltx"
+
+ "github.com/carlosframework/platform/internal/dbdoor/dbdoortest"
+)
+
+var _ litestream.ReplicaClient = (*Client)(nil)
+
+// noNetwork fails the test if the client ever reaches for the wire.
+func noNetwork(t *testing.T) Doer {
+ return func(context.Context, string, http.Header) (*http.Response, error) {
+ t.Error("the client made a request")
+ return nil, errors.New("no network in this test")
+ }
+}
+
+// The claim the whole design rests on: this client cannot corrupt a
+// replica, because it has no code that writes to one. And it refuses
+// without touching the network.
+func TestClientMutatingMethodsAlwaysError(t *testing.T) {
+ c := NewClient(noNetwork(t))
+ ctx := context.Background()
+ if _, err := c.WriteLTXFile(ctx, 0, 1, 2, strings.NewReader("x")); err == nil {
+ t.Error("WriteLTXFile returned nil; this client must never write")
+ }
+ if err := c.DeleteLTXFiles(ctx, []*ltx.FileInfo{{Level: 0, MinTXID: 1, MaxTXID: 1}}); err == nil {
+ t.Error("DeleteLTXFiles returned nil; this client must never delete")
+ }
+ if err := c.DeleteAll(ctx); err == nil {
+ t.Error("DeleteAll returned nil; this client must never delete")
+ }
+}
+
+// Restore type-asserts for ReplicaClientV3 and would take the v0.3 path if
+// this satisfied it. It must not.
+func TestClientIsNotAV3ReplicaClient(t *testing.T) {
+ var c any = NewClient(noNetwork(t))
+ if _, ok := c.(litestream.ReplicaClientV3); ok {
+ t.Fatal("client satisfies ReplicaClientV3; Restore would take the v0.3 path")
+ }
+}
+
+// testDoor serves a file replica over the door's wire shape: the same
+// coordinates, the same Range forms, the same 404 for a missing file. It
+// stands in for the console here; internal/console tests the real door
+// with this package's client.
+type testDoor struct {
+ t *testing.T
+ rc *lsfile.ReplicaClient
+ mu sync.Mutex
+ lists map[int]int // level -> listing count
+ gone map[string]bool
+ opened int
+}
+
+func newTestDoor(t *testing.T, dir string) *testDoor {
+ return &testDoor{t: t, rc: lsfile.NewReplicaClient(dir), lists: map[int]int{}, gone: map[string]bool{}}
+}
+
+func (d *testDoor) do(ctx context.Context, tail string, h http.Header) (*http.Response, error) {
+ rec := httptest.NewRecorder()
+ d.serve(ctx, rec, tail, h)
+ return rec.Result(), nil
+}
+
+func (d *testDoor) serve(ctx context.Context, w http.ResponseWriter, tail string, h http.Header) {
+ parts := strings.Split(strings.TrimPrefix(tail, "/ltx/"), "/")
+ level, err := strconv.Atoi(parts[0])
+ if err != nil {
+ http.NotFound(w, nil)
+ return
+ }
+ if len(parts) == 1 {
+ d.mu.Lock()
+ d.lists[level]++
+ d.mu.Unlock()
+ itr, _ := d.rc.LTXFiles(ctx, level, 0, false)
+ out := ListResponse{Files: []LTXFile{}}
+ for itr.Next() {
+ f := itr.Item()
+ out.Files = append(out.Files, LTXFile{Level: level, MinTXID: uint64(f.MinTXID), MaxTXID: uint64(f.MaxTXID), Size: f.Size, CreatedAt: f.CreatedAt})
+ }
+ itr.Close()
+ json.NewEncoder(w).Encode(out)
+ return
+ }
+ min, max, err := ParseSpan(parts[1])
+ if err != nil {
+ http.NotFound(w, nil)
+ return
+ }
+ d.mu.Lock()
+ gone := d.gone[parts[1]]
+ if gone {
+ delete(d.gone, parts[1]) // gone once: a fresh plan finds it again
+ }
+ d.opened++
+ d.mu.Unlock()
+ if gone {
+ http.NotFound(w, nil)
+ return
+ }
+ off, size, err := ParseRange(h.Get("Range"))
+ if err != nil {
+ w.WriteHeader(http.StatusRequestedRangeNotSatisfiable)
+ return
+ }
+ rc, err := d.rc.OpenLTXFile(ctx, level, ltx.TXID(min), ltx.TXID(max), off, size)
+ if errors.Is(err, os.ErrNotExist) {
+ http.NotFound(w, nil)
+ return
+ }
+ if err != nil {
+ w.WriteHeader(http.StatusInternalServerError)
+ return
+ }
+ defer rc.Close()
+ w.WriteHeader(http.StatusPartialContent)
+ io.Copy(w, rc)
+}
+
+// counting wraps the client and counts every method litestream calls on it.
+type counting struct {
+ *Client
+ mu sync.Mutex
+ counts map[string]int
+}
+
+func (c *counting) inc(m string) {
+ c.mu.Lock()
+ defer c.mu.Unlock()
+ c.counts[m]++
+}
+
+func (c *counting) WriteLTXFile(ctx context.Context, level int, min, max ltx.TXID, r io.Reader) (*ltx.FileInfo, error) {
+ c.inc("WriteLTXFile")
+ return c.Client.WriteLTXFile(ctx, level, min, max, r)
+}
+
+func (c *counting) DeleteLTXFiles(ctx context.Context, a []*ltx.FileInfo) error {
+ c.inc("DeleteLTXFiles")
+ return c.Client.DeleteLTXFiles(ctx, a)
+}
+
+func (c *counting) DeleteAll(ctx context.Context) error {
+ c.inc("DeleteAll")
+ return c.Client.DeleteAll(ctx)
+}
+
+func (c *counting) LTXFiles(ctx context.Context, level int, seek ltx.TXID, useMetadata bool) (ltx.FileIterator, error) {
+ c.inc("LTXFiles")
+ return c.Client.LTXFiles(ctx, level, seek, useMetadata)
+}
+
+func (c *counting) OpenLTXFile(ctx context.Context, level int, min, max ltx.TXID, offset, size int64) (io.ReadCloser, error) {
+ c.inc("OpenLTXFile")
+ return c.Client.OpenLTXFile(ctx, level, min, max, offset, size)
+}
+
+// The methods erroring is not enough: a restore that CALLED one would be
+// relying on an error path. This asserts a full restore never reaches them,
+// which is what catches a litestream upgrade changing the restore path.
+func TestRestoreNeverCallsAMutatingMethod(t *testing.T) {
+ dir := t.TempDir()
+ dbdoortest.Replicate(t, dir, 3)
+ door := newTestDoor(t, dir)
+ c := &counting{Client: NewClient(door.do), counts: map[string]int{}}
+ out := filepath.Join(t.TempDir(), "out.db")
+ opt := litestream.NewRestoreOptions()
+ opt.OutputPath = out
+ if err := litestream.NewReplicaWithClient(nil, c).Restore(context.Background(), opt); err != nil {
+ t.Fatal(err)
+ }
+ if n := c.counts["WriteLTXFile"] + c.counts["DeleteLTXFiles"] + c.counts["DeleteAll"]; n != 0 {
+ t.Fatalf("restore called a mutating method: %v", c.counts)
+ }
+ if c.counts["LTXFiles"] == 0 || c.counts["OpenLTXFile"] == 0 {
+ t.Fatalf("restore did not go through the client at all: %v", c.counts)
+ }
+ if rows := countRows(t, out); rows != 3*dbdoortest.Rows {
+ t.Fatalf("restored %d rows, want %d", rows, 3*dbdoortest.Rows)
+ }
+}
+
+func countRows(t *testing.T, path string) int {
+ t.Helper()
+ db, err := sql.Open("sqlite", "file:"+path+"?mode=ro")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ var n int
+ if err := db.QueryRow(`SELECT count(*) FROM t`).Scan(&n); err != nil {
+ t.Fatal(err)
+ }
+ return n
+}
+
+// seek is not on the wire (wire specific 3): the client filters locally,
+// with litestream's own rule (MinTXID >= seek).
+func TestClientFiltersSeekLocally(t *testing.T) {
+ dir := t.TempDir()
+ dbdoortest.Replicate(t, dir, 3)
+ c := NewClient(newTestDoor(t, dir).do)
+ all := collect(t, c, 0, 0)
+ if len(all) < 2 {
+ t.Fatalf("want several files, got %d", len(all))
+ }
+ seek := all[1].MinTXID
+ got := collect(t, c, 0, seek)
+ if len(got) != len(all)-1 || got[0].MinTXID != seek {
+ t.Fatalf("seek %d = %d files starting %d, want %d starting %d", seek, len(got), got[0].MinTXID, len(all)-1, seek)
+ }
+}
+
+func collect(t *testing.T, c *Client, level int, seek ltx.TXID) []*ltx.FileInfo {
+ t.Helper()
+ itr, err := c.LTXFiles(context.Background(), level, seek, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer itr.Close()
+ var out []*ltx.FileInfo
+ for itr.Next() {
+ out = append(out, itr.Item())
+ }
+ return out
+}
+
+// A 404 on a file is os.ErrNotExist, which is what the whole-Restore retry
+// keys on; any other refusal carries the console's own sentence.
+func TestClientMapsTheDoorsAnswers(t *testing.T) {
+ answer := func(status int, body string) Doer {
+ return func(context.Context, string, http.Header) (*http.Response, error) {
+ rec := httptest.NewRecorder()
+ rec.WriteHeader(status)
+ io.WriteString(rec, body)
+ return rec.Result(), nil
+ }
+ }
+ ctx := context.Background()
+ if _, err := NewClient(answer(404, "404 page not found\n")).OpenLTXFile(ctx, 0, 1, 1, 0, 0); !errors.Is(err, os.ErrNotExist) {
+ t.Errorf("open on 404 = %v, want os.ErrNotExist", err)
+ }
+ _, err := NewClient(answer(409, `{"error":"the data is elsewhere"}`)).LTXFiles(ctx, 0, 0, false)
+ var de *DoorError
+ if !errors.As(err, &de) || de.Status != 409 || de.Message != "the data is elsewhere" {
+ t.Errorf("list on 409 = %v, want a DoorError carrying the sentence", err)
+ }
+ if _, err := NewClient(answer(409, `{"error":"x"}`)).OpenLTXFile(ctx, 0, 1, 1, 0, 0); errors.Is(err, os.ErrNotExist) || !errors.As(err, &de) {
+ t.Errorf("open on 409 = %v, want a DoorError and not ErrNotExist", err)
+ }
+}
+
+// The client sends the S3 client's exact Range header and the coordinate
+// tail, nothing else.
+func TestClientSendsCoordinatesAndRange(t *testing.T) {
+ var gotTail, gotRange string
+ c := NewClient(func(_ context.Context, tail string, h http.Header) (*http.Response, error) {
+ gotTail, gotRange = tail, h.Get("Range")
+ rec := httptest.NewRecorder()
+ rec.WriteHeader(http.StatusPartialContent)
+ return rec.Result(), nil
+ })
+ rc, err := c.OpenLTXFile(context.Background(), 3, 16, 32, 512, 100)
+ if err != nil {
+ t.Fatal(err)
+ }
+ rc.Close()
+ if gotTail != GetTail(3, 16, 32) || gotRange != RangeHeader(512, 100) {
+ t.Fatalf("tail %q range %q, want %q %q", gotTail, gotRange, GetTail(3, 16, 32), RangeHeader(512, 100))
+ }
+}
+
+// Retry-After is capped at a minute however large the door's number, and a
+// number too big to multiply into a Duration must not wrap round to a
+// short or negative wait.
+func TestClientCapsRetryAfter(t *testing.T) {
+ for _, v := range []string{"61", "3600", "18446744074", "9223372036", "9223372037"} {
+ do := func(context.Context, string, http.Header) (*http.Response, error) {
+ rec := httptest.NewRecorder()
+ rec.Header().Set("Retry-After", v)
+ rec.WriteHeader(http.StatusTooManyRequests)
+ io.WriteString(rec, `{"error":"busy"}`)
+ return rec.Result(), nil
+ }
+ _, err := NewClient(do).OpenLTXFile(context.Background(), 0, 1, 1, 0, 0)
+ var de *DoorError
+ if !errors.As(err, &de) || de.RetryAfter != time.Minute {
+ t.Errorf("Retry-After: %s gave %v, want the one-minute cap", v, de.RetryAfter)
+ }
+ }
+}
diff --git a/internal/dbdoor/coords.go b/internal/dbdoor/coords.go
new file mode 100644
index 000000000..67aba1ced
--- /dev/null
+++ b/internal/dbdoor/coords.go
@@ -0,0 +1,133 @@
+// Package dbdoor is the wire shape of the `carlos db` replica door, shared
+// by the console that serves it and the CLI that reads it (spec
+// 2026-09-02 §6), plus the CLI's read-only replica client and its restore.
+//
+// It is a LEAF package for the reason internal/dbverify is one: the server
+// (internal/console) and the client (the CLI's main package) must agree on
+// one shape, and neither can import the other.
+//
+// Files are addressed by COORDINATE (level plus TXID range), never by key.
+// That is not decoration: lock.json lives under the same prefix as the LTX
+// files and is not a member's to fetch, and a coordinate has no way to name
+// it. See spec §6 "The door".
+package dbdoor
+
+import (
+ "errors"
+ "fmt"
+ "net/url"
+ "strconv"
+ "strings"
+ "time"
+)
+
+// MaxLevel is the highest level a list may name. litestream's
+// CalcRestorePlan asks for SnapshotLevel (9) first, then 8 down to 0. The
+// activator writes only 0-3 and 9, so most of that range is legitimately
+// EMPTY, and empty is an empty list, never a 404, or the restore stops at
+// the first gap.
+const MaxLevel = 9
+
+// LTXFile is one replica file as the door reports it. Size, MinTXID and
+// MaxTXID are MANDATORY: Restore rejects Size < ltx.HeaderSize and
+// ResumableReader uses Size to detect a premature EOF.
+type LTXFile struct {
+ Level int `json:"level"`
+ MinTXID uint64 `json:"min_txid"`
+ MaxTXID uint64 `json:"max_txid"`
+ Size int64 `json:"size"`
+ CreatedAt time.Time `json:"created_at"`
+}
+
+// ListResponse is ordered by MinTXID ASCENDING: restoreLevelCursor assumes
+// filename order, which S3 gives because the names are zero-padded hex.
+type ListResponse struct {
+ Files []LTXFile `json:"files"`
+}
+
+// Base is the door's root for one instance: everything below it is a
+// coordinate. The account segment is passed through verbatim (sqid or
+// name), exactly as every other /api/cli/apps/ path does.
+func Base(acct, app, host string) string {
+ return fmt.Sprintf("/api/cli/apps/%s/%s/db/%s",
+ url.PathEscape(acct), url.PathEscape(app), url.PathEscape(host))
+}
+
+// ListTail and GetTail are the coordinate halves of a path, relative to
+// Base. The CLI's client works in tails, because the account segment and its
+// ?exact=sqid switch belong to the CLI's own path builder.
+func ListTail(level int) string { return fmt.Sprintf("/ltx/%d", level) }
+
+func GetTail(level int, minTXID, maxTXID uint64) string {
+ return fmt.Sprintf("/ltx/%d/%s", level, Span(minTXID, maxTXID))
+}
+
+func ListPath(acct, app, host string, level int) string {
+ return Base(acct, app, host) + ListTail(level)
+}
+
+func GetPath(acct, app, host string, level int, minTXID, maxTXID uint64) string {
+ return Base(acct, app, host) + GetTail(level, minTXID, maxTXID)
+}
+
+// Span renders a TXID range as one path segment.
+func Span(minTXID, maxTXID uint64) string {
+ return strconv.FormatUint(minTXID, 10) + "-" + strconv.FormatUint(maxTXID, 10)
+}
+
+var errSpan = errors.New("dbdoor: not a txid span")
+
+// ParseSpan is Span's inverse. TXIDs start at 1 and a span never runs
+// backwards; anything else is not a coordinate.
+func ParseSpan(s string) (minTXID, maxTXID uint64, err error) {
+ a, b, ok := strings.Cut(s, "-")
+ if !ok {
+ return 0, 0, errSpan
+ }
+ minTXID, err1 := strconv.ParseUint(a, 10, 64)
+ maxTXID, err2 := strconv.ParseUint(b, 10, 64)
+ if err1 != nil || err2 != nil || minTXID == 0 || maxTXID < minTXID {
+ return 0, 0, errSpan
+ }
+ return minTXID, maxTXID, nil
+}
+
+// RangeHeader mirrors litestream's own S3 client byte-for-byte
+// (s3/replica_client.go:646-651): open-ended when size is 0, inclusive end
+// otherwise.
+func RangeHeader(offset, size int64) string {
+ if size == 0 {
+ return fmt.Sprintf("bytes=%d-", offset)
+ }
+ return fmt.Sprintf("bytes=%d-%d", offset, offset+size-1)
+}
+
+var errRange = errors.New("dbdoor: unsupported range")
+
+// ParseRange is RangeHeader's inverse, and accepts only what RangeHeader
+// produces. An empty header is the whole file.
+func ParseRange(h string) (offset, size int64, err error) {
+ if h == "" {
+ return 0, 0, nil
+ }
+ spec, ok := strings.CutPrefix(h, "bytes=")
+ if !ok {
+ return 0, 0, errRange
+ }
+ a, b, ok := strings.Cut(spec, "-")
+ if !ok || a == "" {
+ return 0, 0, errRange
+ }
+ offset, err = strconv.ParseInt(a, 10, 64)
+ if err != nil || offset < 0 {
+ return 0, 0, errRange
+ }
+ if b == "" {
+ return offset, 0, nil
+ }
+ end, err := strconv.ParseInt(b, 10, 64)
+ if err != nil || end < offset {
+ return 0, 0, errRange
+ }
+ return offset, end - offset + 1, nil
+}
diff --git a/internal/dbdoor/coords_test.go b/internal/dbdoor/coords_test.go
new file mode 100644
index 000000000..0acb17a50
--- /dev/null
+++ b/internal/dbdoor/coords_test.go
@@ -0,0 +1,77 @@
+package dbdoor
+
+import (
+ "strings"
+ "testing"
+)
+
+// Range semantics must match litestream's S3 client exactly
+// (s3/replica_client.go:646-651), or a resumed read returns the wrong bytes.
+func TestRangeHeaderMatchesTheS3Client(t *testing.T) {
+ cases := []struct {
+ name string
+ offset, size int64
+ want string
+ }{
+ {"whole file", 0, 0, "bytes=0-"},
+ {"open-ended from an offset", 512, 0, "bytes=512-"},
+ {"sized read", 0, 100, "bytes=0-99"},
+ {"sized read at an offset", 512, 100, "bytes=512-611"},
+ }
+ for _, tc := range cases {
+ t.Run(tc.name, func(t *testing.T) {
+ if got := RangeHeader(tc.offset, tc.size); got != tc.want {
+ t.Errorf("RangeHeader(%d, %d) = %q, want %q", tc.offset, tc.size, got, tc.want)
+ }
+ })
+ }
+}
+
+// ParseRange is RangeHeader's inverse, used by the door. Anything it does not
+// produce is refused rather than guessed at.
+func TestParseRangeInvertsRangeHeader(t *testing.T) {
+ for _, c := range []struct{ offset, size int64 }{{0, 0}, {512, 0}, {0, 100}, {512, 100}} {
+ off, size, err := ParseRange(RangeHeader(c.offset, c.size))
+ if err != nil || off != c.offset || size != c.size {
+ t.Errorf("ParseRange(RangeHeader(%d, %d)) = %d, %d, %v", c.offset, c.size, off, size, err)
+ }
+ }
+ if off, size, err := ParseRange(""); err != nil || off != 0 || size != 0 {
+ t.Errorf("ParseRange(\"\") = %d, %d, %v, want the whole file", off, size, err)
+ }
+ for _, bad := range []string{"bytes=-5", "bytes=5-2", "bytes=0-1,3-4", "items=0-", "bytes=x-", "bytes=0-y"} {
+ if _, _, err := ParseRange(bad); err == nil {
+ t.Errorf("ParseRange(%q) accepted", bad)
+ }
+ }
+}
+
+// Coordinates, never keys: the caller names a level and a TXID range, and
+// the server decides what object that is. lock.json has no coordinate.
+func TestGetPathIsCoordinateAddressed(t *testing.T) {
+ got := GetPath("bab", "titogo", "a.bab.oncarlos.com", 3, 16, 32)
+ for _, want := range []string{"bab", "titogo", "a.bab.oncarlos.com", "/3/", "16", "32"} {
+ if !strings.Contains(got, want) {
+ t.Fatalf("GetPath = %q, missing %q", got, want)
+ }
+ }
+ if strings.Contains(got, ".ltx") || strings.Contains(got, "lock") {
+ t.Fatalf("GetPath = %q leaks the key layout", got)
+ }
+ if !strings.HasSuffix(got, GetTail(3, 16, 32)) || !strings.HasSuffix(ListPath("bab", "titogo", "h.example.com", 3), ListTail(3)) {
+ t.Fatalf("full paths and tails disagree")
+ }
+}
+
+// The span segment round-trips, and nothing that is not a span parses.
+func TestParseSpanRoundTripsAndRefusesTheRest(t *testing.T) {
+ min, max, err := ParseSpan(Span(16, 32))
+ if err != nil || min != 16 || max != 32 {
+ t.Fatalf("ParseSpan(Span(16, 32)) = %d, %d, %v", min, max, err)
+ }
+ for _, bad := range []string{"", "16", "16-", "-32", "32-16", "0-1", "a-b", "lock.json", "16-32-48", "+1-2"} {
+ if _, _, err := ParseSpan(bad); err == nil {
+ t.Errorf("ParseSpan(%q) accepted", bad)
+ }
+ }
+}
diff --git a/internal/dbdoor/dbdoortest/replica.go b/internal/dbdoor/dbdoortest/replica.go
new file mode 100644
index 000000000..c3dce59b6
--- /dev/null
+++ b/internal/dbdoor/dbdoortest/replica.go
@@ -0,0 +1,78 @@
+// Package dbdoortest builds real litestream replicas for the db door's tests,
+// on both sides of the wire (internal/console serves one, internal/dbdoor and
+// the CLI restore one). A fake replica would prove only that the fake agrees
+// with itself; these are written by litestream's own replication code, in
+// the layout its file client reads, so a restore through the door is the
+// restore production runs.
+//
+// Test support only: nothing outside a _test.go file may import it.
+package dbdoortest
+
+import (
+ "context"
+ "database/sql"
+ "fmt"
+ "path/filepath"
+ "testing"
+ "time"
+
+ "github.com/benbjohnson/litestream"
+ lsfile "github.com/benbjohnson/litestream/file"
+ _ "modernc.org/sqlite"
+)
+
+// Rows is how many rows each batch inserts into table t.
+const Rows = 10
+
+// Replicate creates a WAL-mode SQLite database with a table t(id, body) and
+// replicates it into dir in `batches` separate syncs, so the replica holds
+// several L0 files rather than one. It returns the source database's path;
+// the source holds batches*Rows rows when it returns.
+func Replicate(t testing.TB, dir string, batches int) string {
+ t.Helper()
+ src := filepath.Join(t.TempDir(), "source.db")
+ db, err := sql.Open("sqlite", src)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ if _, err := db.Exec(`PRAGMA journal_mode=WAL; CREATE TABLE t (id INTEGER PRIMARY KEY, body TEXT)`); err != nil {
+ t.Fatal(err)
+ }
+ for b := 0; b < batches; b++ {
+ for i := 0; i < Rows; i++ {
+ if _, err := db.Exec(`INSERT INTO t (body) VALUES (?)`, fmt.Sprintf("row-%d-%d", b, i)); err != nil {
+ t.Fatal(err)
+ }
+ }
+ Sync(t, src, dir)
+ }
+ return src
+}
+
+// Sync replicates the source's committed state into dir once, the way the
+// lease package's Checkpoint does: open, one explicit sync, close.
+func Sync(t testing.TB, src, dir string) {
+ t.Helper()
+ ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+ defer cancel()
+ ldb := litestream.NewDB(src)
+ ldb.SetMetaPath(filepath.Join(filepath.Dir(src), ".dbdoortest-litestream"))
+ ldb.MonitorInterval = 0
+ ldb.Replica = litestream.NewReplicaWithClient(ldb, lsfile.NewReplicaClient(dir))
+ ldb.Replica.MonitorEnabled = false
+ store := litestream.NewStore([]*litestream.DB{ldb}, litestream.CompactionLevels{{Level: 0}})
+ store.CompactionMonitorEnabled = false
+ store.L0RetentionCheckInterval = 0
+ store.SetRetentionEnabled(false)
+ if err := store.Open(ctx); err != nil {
+ t.Fatal(err)
+ }
+ syncErr := ldb.SyncAndWait(ctx)
+ if err := store.Close(ctx); err != nil {
+ t.Fatal(err)
+ }
+ if syncErr != nil {
+ t.Fatal(syncErr)
+ }
+}
diff --git a/internal/dbdoor/dbdoortest/replica_test.go b/internal/dbdoor/dbdoortest/replica_test.go
new file mode 100644
index 000000000..b48813c9c
--- /dev/null
+++ b/internal/dbdoor/dbdoortest/replica_test.go
@@ -0,0 +1,49 @@
+package dbdoortest
+
+import (
+ "context"
+ "database/sql"
+ "path/filepath"
+ "testing"
+
+ "github.com/benbjohnson/litestream"
+ lsfile "github.com/benbjohnson/litestream/file"
+)
+
+// The helper is only worth having if litestream itself can restore what it
+// wrote, with every batch present, from more than one L0 file.
+func TestReplicateWritesARestorableMultiFileReplica(t *testing.T) {
+ dir := t.TempDir()
+ Replicate(t, dir, 3)
+ c := lsfile.NewReplicaClient(dir)
+ itr, err := c.LTXFiles(context.Background(), 0, 0, false)
+ if err != nil {
+ t.Fatal(err)
+ }
+ n := 0
+ for itr.Next() {
+ n++
+ }
+ itr.Close()
+ if n < 2 {
+ t.Fatalf("replica holds %d L0 files, want several", n)
+ }
+ out := filepath.Join(t.TempDir(), "out.db")
+ opt := litestream.NewRestoreOptions()
+ opt.OutputPath = out
+ if err := litestream.NewReplicaWithClient(nil, c).Restore(context.Background(), opt); err != nil {
+ t.Fatal(err)
+ }
+ db, err := sql.Open("sqlite", "file:"+out+"?mode=ro")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ var rows int
+ if err := db.QueryRow(`SELECT count(*) FROM t`).Scan(&rows); err != nil {
+ t.Fatal(err)
+ }
+ if rows != 3*Rows {
+ t.Fatalf("restored %d rows, want %d", rows, 3*Rows)
+ }
+}
diff --git a/internal/dbdoor/restore.go b/internal/dbdoor/restore.go
new file mode 100644
index 000000000..85d5ee802
--- /dev/null
+++ b/internal/dbdoor/restore.go
@@ -0,0 +1,340 @@
+package dbdoor
+
+import (
+ "bytes"
+ "context"
+ "errors"
+ "fmt"
+ "io"
+ "net/http"
+ "os"
+ "path/filepath"
+ "sync"
+ "time"
+
+ "github.com/benbjohnson/litestream"
+ "github.com/superfly/ltx"
+
+ "github.com/carlosframework/platform/internal/lease"
+)
+
+var (
+ // ErrNothingReplicated: the replica holds no restorable state yet.
+ // litestream says so two ways (ErrNoSnapshots, ErrTxNotAvailable), the
+ // pair lease.Restore already treats as "nothing to restore".
+ ErrNothingReplicated = errors.New("dbdoor: nothing has been replicated yet")
+ // ErrCheckFailed: the copy restored, but lease.CheckDB refused it. It
+ // is never handed over.
+ ErrCheckFailed = errors.New("dbdoor: the restored copy failed its check")
+)
+
+// Restored is a private copy of a replica on this machine.
+type Restored struct {
+ // Path is the restored database. It lives inside Dir.
+ Path string
+ // Dir is the private 0700 directory holding Path and any -wal/-shm a
+ // read-only open makes. The caller removes it.
+ Dir string
+ // TXID is the newest replicated transaction in the copy, and At is
+ // that transaction's time from its LTX header: the time of the write,
+ // not of a later compaction (spec §8.1).
+ TXID uint64
+ At time.Time
+}
+
+// restoredName is the copy's file name inside its private directory.
+const restoredName = "replica.db"
+
+// checkDB is lease.CheckDBContext, the one check a disaster recovery leans on. A
+// variable only so a test can make it refuse.
+var checkDB = lease.CheckDBContext
+
+// RestoreTo restores the replica c reads into a new 0700 directory under
+// parent (os.TempDir() when empty) and proves it with lease.CheckDB.
+//
+// The three conditions spec §12 rests on hold here: c is not a
+// ReplicaClientV3, RestoreOptions.Follow is never set, and CheckDB runs on
+// the result rather than relying on litestream's own IntegrityCheck.
+//
+// A planned file can vanish under the restore when the writer's L0
+// retention fires, and litestream's ResumableReader treats that as fatal.
+// So the retry wraps the WHOLE Restore, once: a fresh plan is the only
+// recovery. On any failure the directory is removed, so no copy of
+// production data is left behind.
+func RestoreTo(ctx context.Context, c *Client, parent string) (*Restored, error) {
+ dir, err := os.MkdirTemp(parent, "carlos-db-")
+ if err != nil {
+ return nil, err
+ }
+ // MkdirTemp already creates 0700; say it anyway, because the privacy of
+ // everything restored rests on this one mode.
+ if err := os.Chmod(dir, 0o700); err != nil {
+ os.RemoveAll(dir)
+ return nil, err
+ }
+ out := filepath.Join(dir, restoredName)
+ var rec *recorder
+ for attempt := 0; ; attempt++ {
+ rec = &recorder{Client: c, dir: dir}
+ err = restoreOnce(ctx, rec, out)
+ if err == nil {
+ break
+ }
+ removeCopy(out)
+ vanished := errors.Is(err, os.ErrNotExist) || rec.sawVanish()
+ if attempt == 0 && vanished && ctx.Err() == nil {
+ continue
+ }
+ os.RemoveAll(dir)
+ // A stop reaches litestream's local half as a broken read, which
+ // its compactor reports without wrapping. Name the real cause.
+ if cerr := ctx.Err(); cerr != nil {
+ return nil, cerr
+ }
+ switch {
+ case errors.Is(err, litestream.ErrNoSnapshots) || errors.Is(err, litestream.ErrTxNotAvailable):
+ return nil, ErrNothingReplicated
+ case vanished && !errors.Is(err, os.ErrNotExist):
+ return nil, fmt.Errorf("%w: %w", os.ErrNotExist, err)
+ }
+ return nil, err
+ }
+ if err := ctx.Err(); err != nil {
+ os.RemoveAll(dir)
+ return nil, err
+ }
+ if err := checkDB(ctx, out); err != nil {
+ os.RemoveAll(dir)
+ if cerr := ctx.Err(); cerr != nil {
+ return nil, cerr
+ }
+ return nil, fmt.Errorf("%w: %v", ErrCheckFailed, err)
+ }
+ // A stop that lands as the check finishes still withdraws the copy.
+ if err := ctx.Err(); err != nil {
+ os.RemoveAll(dir)
+ return nil, err
+ }
+ txid, at := rec.newest()
+ return &Restored{Path: out, Dir: dir, TXID: txid, At: at}, nil
+}
+
+func restoreOnce(ctx context.Context, client litestream.ReplicaClient, out string) error {
+ opt := litestream.NewRestoreOptions()
+ opt.OutputPath = out
+ // Follow stays false (spec §12 condition ii); IntegrityCheck is left at
+ // litestream's default and CheckDB runs after (condition iii).
+ return litestream.NewReplicaWithClient(nil, client).Restore(ctx, opt)
+}
+
+func removeCopy(out string) {
+ for _, p := range []string{out, out + ".tmp", out + "-wal", out + "-shm"} {
+ _ = os.Remove(p)
+ }
+}
+
+// recorder sees every file a Restore opens, for two reasons: a file that
+// answered os.ErrNotExist is what makes the restore worth a second plan,
+// whatever litestream wrapped it in on the way out; and the header of the
+// newest file opened is the copy's "as of" (R1).
+type recorder struct {
+ *Client
+ dir string // the private directory each file is downloaded into
+ mu sync.Mutex
+ vanished bool
+ maxTXID ltx.TXID
+ header *bytes.Buffer // first ltx.HeaderSize bytes of the maxTXID file
+}
+
+func (r *recorder) OpenLTXFile(ctx context.Context, level int, minTXID, maxTXID ltx.TXID, offset, size int64) (io.ReadCloser, error) {
+ // The first open goes through download's loop too, so a full door's
+ // 429 waits out its Retry-After here rather than failing back to
+ // litestream, whose own quick retries ignore it.
+ rc, err := r.download(ctx, nil, level, minTXID, maxTXID, offset, size)
+ if err != nil {
+ return nil, err
+ }
+ if offset != 0 {
+ return rc, nil
+ }
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ if maxTXID < r.maxTXID {
+ return rc, nil
+ }
+ r.maxTXID = maxTXID
+ r.header = &bytes.Buffer{}
+ return &headerTee{ReadCloser: rc, buf: r.header}, nil
+}
+
+// dbDownloadMaxStalls is how many reopens in a row may deliver no byte
+// before a download gives up. Progress resets the count, so a slow but
+// moving link always finishes.
+const dbDownloadMaxStalls = 5
+
+// download reads one file whole into the private directory and closes the
+// stream before the restore reads a byte of it. litestream's compactor
+// holds every planned file open until the restore ends, so streaming
+// straight through would hold one door stream per planned file, and the
+// door bounds a person's concurrent streams (apidb.go). Downloaded, a
+// restore holds one stream at a time.
+//
+// A connection that drops part way is resumed here, from the bytes already
+// kept, rather than handed to litestream: its resumable reader would start
+// the file again from the requested offset, and a large file over a flaky
+// link would never finish. The file goes when the restore closes it; on
+// failure, with the whole directory.
+func (r *recorder) download(ctx context.Context, rc io.ReadCloser, level int, minTXID, maxTXID ltx.TXID, offset, size int64) (io.ReadCloser, error) {
+ f, err := os.CreateTemp(r.dir, "ltx-*")
+ if err != nil {
+ rc.Close()
+ return nil, err
+ }
+ fail := func(err error) (io.ReadCloser, error) {
+ f.Close()
+ os.Remove(f.Name())
+ return nil, err
+ }
+ var written int64
+ stalls, wait := 0, dbDownloadBackoff
+ for {
+ if rc != nil {
+ n, cerr := io.Copy(f, rc)
+ rc.Close()
+ rc = nil
+ written += n
+ if cerr == nil {
+ break
+ }
+ if n > 0 {
+ stalls, wait = 0, dbDownloadBackoff
+ } else if stalls++; stalls >= dbDownloadMaxStalls {
+ return fail(cerr)
+ }
+ }
+ if ctx.Err() != nil {
+ return fail(ctx.Err())
+ }
+ var rest int64 // 0: to the end, as RangeHeader reads it
+ if size > 0 {
+ if rest = size - written; rest <= 0 {
+ break
+ }
+ }
+ next, err := r.Client.OpenLTXFile(ctx, level, minTXID, maxTXID, offset+written, rest)
+ if err == nil {
+ rc = next
+ continue
+ }
+ switch {
+ case errors.Is(err, os.ErrNotExist):
+ r.mu.Lock()
+ r.vanished = true
+ r.mu.Unlock()
+ return fail(err)
+ case ctx.Err() != nil:
+ return fail(ctx.Err())
+ case !transientOpen(err):
+ return fail(err)
+ }
+ // A busy door or a dropped connection: wait and ask again for the
+ // same remaining range, keeping every byte already here. It counts
+ // as an attempt that delivered nothing.
+ if stalls++; stalls >= dbDownloadMaxStalls {
+ return fail(err)
+ }
+ d := wait
+ var door *DoorError
+ if errors.As(err, &door) && door.RetryAfter > 0 {
+ d = door.RetryAfter
+ }
+ wait = min(wait*2, dbDownloadBackoffMax)
+ select {
+ case <-ctx.Done():
+ return fail(ctx.Err())
+ case <-time.After(d):
+ }
+ }
+ if _, err := f.Seek(0, io.SeekStart); err != nil {
+ return fail(err)
+ }
+ return &downloaded{ctx: ctx, File: f}, nil
+}
+
+// downloaded is a file read once and removed on Close.
+type downloaded struct {
+ ctx context.Context
+ *os.File
+}
+
+// Read checks the restore's context first. Once every file is downloaded,
+// litestream's compaction reads only these files and checks no context of
+// its own, so this is where Ctrl-C reaches it.
+func (d *downloaded) Read(p []byte) (int, error) {
+ if err := d.ctx.Err(); err != nil {
+ return 0, err
+ }
+ return d.File.Read(p)
+}
+
+func (d *downloaded) Close() error {
+ err := d.File.Close()
+ if rerr := os.Remove(d.File.Name()); err == nil {
+ err = rerr
+ }
+ return err
+}
+
+func (r *recorder) sawVanish() bool {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ return r.vanished
+}
+
+// newest returns the copy's newest transaction and the time its LTX header
+// records. A header that cannot be read leaves At zero rather than guessed.
+func (r *recorder) newest() (uint64, time.Time) {
+ r.mu.Lock()
+ defer r.mu.Unlock()
+ var h ltx.Header
+ if r.header == nil || r.header.Len() < ltx.HeaderSize || h.UnmarshalBinary(r.header.Bytes()[:ltx.HeaderSize]) != nil {
+ return uint64(r.maxTXID), time.Time{}
+ }
+ return uint64(r.maxTXID), time.UnixMilli(h.Timestamp).UTC()
+}
+
+// headerTee copies the first ltx.HeaderSize bytes read through it.
+type headerTee struct {
+ io.ReadCloser
+ buf *bytes.Buffer
+}
+
+func (t *headerTee) Read(p []byte) (int, error) {
+ n, err := t.ReadCloser.Read(p)
+ if need := ltx.HeaderSize - t.buf.Len(); need > 0 && n > 0 {
+ t.buf.Write(p[:min(n, need)])
+ }
+ return n, err
+}
+
+// dbDownloadBackoff is the first wait before retrying a reopen that met a
+// busy door or a dropped connection; each further wait doubles, up to
+// dbDownloadBackoffMax. A 429's Retry-After overrides it.
+var dbDownloadBackoff = 250 * time.Millisecond
+
+const dbDownloadBackoffMax = 8 * time.Second
+
+// transientOpen reports a reopen failure worth waiting out: the door busy
+// or briefly unavailable (429, 502, 503, 504), or no answer at all (a
+// transport error). Any other refusal from the door is final.
+func transientOpen(err error) bool {
+ var door *DoorError
+ if errors.As(err, &door) {
+ switch door.Status {
+ case http.StatusTooManyRequests, http.StatusBadGateway, http.StatusServiceUnavailable, http.StatusGatewayTimeout:
+ return true
+ }
+ return false
+ }
+ return true
+}
diff --git a/internal/dbdoor/restore_test.go b/internal/dbdoor/restore_test.go
new file mode 100644
index 000000000..3836ed4df
--- /dev/null
+++ b/internal/dbdoor/restore_test.go
@@ -0,0 +1,523 @@
+package dbdoor
+
+import (
+ "context"
+ "errors"
+ "io"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "sync"
+ "sync/atomic"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/dbdoor/dbdoortest"
+)
+
+// litestream creates <out>.tmp itself with os.Create, at the process umask,
+// so a 0600 file cannot be pre-created for it. The containing directory is
+// the only thing that can be private, and it must also confine the -wal
+// and -shm a read-only open still needs.
+func TestRestoreToUsesAPrivateDirectory(t *testing.T) {
+ src := t.TempDir()
+ dbdoortest.Replicate(t, src, 2)
+ parent := t.TempDir()
+ got, err := RestoreTo(context.Background(), NewClient(newTestDoor(t, src).do), parent)
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(got.Dir)
+ if filepath.Dir(got.Path) != got.Dir || filepath.Dir(got.Dir) != parent {
+ t.Fatalf("path %s, dir %s: want the copy inside a fresh directory under %s", got.Path, got.Dir, parent)
+ }
+ fi, err := os.Stat(got.Dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if perm := fi.Mode().Perm(); perm != 0o700 {
+ t.Fatalf("restore directory is %04o, want 0700", perm)
+ }
+ if rows := countRows(t, got.Path); rows != 2*dbdoortest.Rows {
+ t.Fatalf("restored %d rows, want %d", rows, 2*dbdoortest.Rows)
+ }
+}
+
+// R1: the handover names the newest replicated transaction and its time,
+// and the time is the LTX header's, the time of the write itself.
+func TestRestoreToReportsTheNewestTransaction(t *testing.T) {
+ src := t.TempDir()
+ before := time.Now().Add(-time.Second)
+ dbdoortest.Replicate(t, src, 3)
+ door := newTestDoor(t, src)
+ c := NewClient(door.do)
+ files := collect(t, c, 0, 0)
+ want := files[len(files)-1].MaxTXID
+ got, err := RestoreTo(context.Background(), c, t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(got.Dir)
+ if got.TXID != uint64(want) {
+ t.Errorf("TXID = %d, want %d", got.TXID, want)
+ }
+ if got.At.Before(before) || got.At.After(time.Now().Add(time.Second)) {
+ t.Errorf("At = %v, want the time of the last write (around now)", got.At)
+ }
+}
+
+// A planned file can vanish under the restore when the writer's L0
+// retention fires. ResumableReader treats ErrNotExist as FATAL and does not
+// retry, so the retry has to wrap the whole Restore: a fresh plan is the
+// only recovery.
+func TestRestoreToRetriesTheWholeRestoreOnce(t *testing.T) {
+ src := t.TempDir()
+ dbdoortest.Replicate(t, src, 2)
+ door := newTestDoor(t, src)
+ first := collect(t, NewClient(door.do), 0, 0)[0]
+ door.gone[Span(uint64(first.MinTXID), uint64(first.MaxTXID))] = true
+ door.lists = map[int]int{}
+ got, err := RestoreTo(context.Background(), NewClient(door.do), t.TempDir())
+ if err != nil {
+ t.Fatalf("RestoreTo with a file vanishing once = %v, want success on a fresh plan", err)
+ }
+ defer os.RemoveAll(got.Dir)
+ if door.lists[9] != 2 {
+ t.Fatalf("snapshot level listed %d times, want 2 (one plan per Restore)", door.lists[9])
+ }
+ if rows := countRows(t, got.Path); rows != 2*dbdoortest.Rows {
+ t.Fatalf("restored %d rows, want %d", rows, 2*dbdoortest.Rows)
+ }
+}
+
+// Once, not forever: a file that keeps vanishing is a failure to report,
+// and the private directory goes with it.
+func TestRestoreToRetriesOnlyOnce(t *testing.T) {
+ src := t.TempDir()
+ dbdoortest.Replicate(t, src, 2)
+ door := newTestDoor(t, src)
+ first := collect(t, NewClient(door.do), 0, 0)[0]
+ span := Span(uint64(first.MinTXID), uint64(first.MaxTXID))
+ c := NewClient(func(ctx context.Context, tail string, h http.Header) (*http.Response, error) {
+ door.mu.Lock()
+ door.gone[span] = true
+ door.mu.Unlock()
+ return door.do(ctx, tail, h)
+ })
+ parent := t.TempDir()
+ if _, err := RestoreTo(context.Background(), c, parent); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("RestoreTo = %v, want the vanished file reported", err)
+ }
+ assertEmptyDir(t, parent)
+}
+
+// "Nothing replicated yet" is its own answer, for both of the errors
+// litestream gives it (spec §10), and leaves nothing behind.
+func TestRestoreToNamesAnEmptyReplica(t *testing.T) {
+ parent := t.TempDir()
+ _, err := RestoreTo(context.Background(), NewClient(newTestDoor(t, t.TempDir()).do), parent)
+ if !errors.Is(err, ErrNothingReplicated) {
+ t.Fatalf("RestoreTo of an empty replica = %v, want ErrNothingReplicated", err)
+ }
+ assertEmptyDir(t, parent)
+}
+
+// A copy that fails lease.CheckDB is never handed over (spec §10). A
+// replica litestream wrote always holds its own _litestream_seq table and
+// a corrupt one fails inside Restore, so the check's refusal is injected:
+// what is under test is that RestoreTo honours it.
+func TestRestoreToRefusesACopyThatFailsTheCheck(t *testing.T) {
+ src := t.TempDir()
+ dbdoortest.Replicate(t, src, 1)
+ var checked string
+ defer func(old func(context.Context, string) error) { checkDB = old }(checkDB)
+ checkDB = func(_ context.Context, path string) error {
+ checked = path
+ return errors.New("integrity_check: page 2 is never used")
+ }
+ parent := t.TempDir()
+ _, err := RestoreTo(context.Background(), NewClient(newTestDoor(t, src).do), parent)
+ if !errors.Is(err, ErrCheckFailed) {
+ t.Fatalf("RestoreTo of a copy failing its check = %v, want ErrCheckFailed", err)
+ }
+ if filepath.Base(checked) != restoredName {
+ t.Fatalf("checked %q, want the restored copy", checked)
+ }
+ assertEmptyDir(t, parent)
+}
+
+func assertEmptyDir(t *testing.T, dir string) {
+ t.Helper()
+ ents, err := os.ReadDir(dir)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(ents) != 0 {
+ t.Fatalf("%s holds %d entries after a failed restore, want none (no copy of production left behind)", dir, len(ents))
+ }
+}
+
+// countingBody tracks how many door responses are open at once.
+type countingBody struct {
+ io.ReadCloser
+ open *atomic.Int32
+ once sync.Once
+}
+
+func (b *countingBody) Close() error {
+ b.once.Do(func() { b.open.Add(-1) })
+ return b.ReadCloser.Close()
+}
+
+// The door bounds a person's concurrent file streams, and litestream's
+// compactor holds every planned file open until the restore ends. Streaming
+// straight through would need one slot per file, so RestoreTo downloads each
+// file whole into its private directory before handing it on: one stream at
+// a time, whatever the plan's length.
+func TestRestoreToHoldsOneStreamAtATime(t *testing.T) {
+ src := t.TempDir()
+ dbdoortest.Replicate(t, src, 4)
+ door := newTestDoor(t, src)
+ var open, peak atomic.Int32
+ do := func(ctx context.Context, tail string, h http.Header) (*http.Response, error) {
+ res, err := door.do(ctx, tail, h)
+ if err != nil || !strings.Contains(strings.TrimPrefix(tail, "/ltx/"), "/") {
+ return res, err // listings are read and closed at once
+ }
+ n := open.Add(1)
+ for p := peak.Load(); n > p && !peak.CompareAndSwap(p, n); p = peak.Load() {
+ }
+ res.Body = &countingBody{ReadCloser: res.Body, open: &open}
+ return res, nil
+ }
+ got, err := RestoreTo(context.Background(), NewClient(do), t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(got.Dir)
+ if p := peak.Load(); p != 1 {
+ t.Fatalf("restore held %d file streams open at once, want 1", p)
+ }
+ if rows := countRows(t, got.Path); rows != 4*dbdoortest.Rows {
+ t.Fatalf("restored %d rows, want %d", rows, 4*dbdoortest.Rows)
+ }
+ if ents, _ := os.ReadDir(got.Dir); len(ents) > 3 {
+ t.Fatalf("downloaded files left in the private directory: %v", ents)
+ }
+}
+
+// flakyBody hands over at most limit bytes and then fails the way a reset
+// connection does.
+type flakyBody struct {
+ io.ReadCloser
+ left int
+}
+
+func (b *flakyBody) Read(p []byte) (int, error) {
+ if b.left <= 0 {
+ return 0, io.ErrUnexpectedEOF
+ }
+ if len(p) > b.left {
+ p = p[:b.left]
+ }
+ n, err := b.ReadCloser.Read(p)
+ b.left -= n
+ return n, err
+}
+
+// flaky wraps a door so every file response breaks after limit bytes.
+func flaky(door *testDoor, limit int, gets *atomic.Int32) Doer {
+ return func(ctx context.Context, tail string, h http.Header) (*http.Response, error) {
+ res, err := door.do(ctx, tail, h)
+ if err != nil || !strings.Contains(strings.TrimPrefix(tail, "/ltx/"), "/") {
+ return res, err
+ }
+ gets.Add(1)
+ res.Body = &flakyBody{ReadCloser: res.Body, left: limit}
+ return res, nil
+ }
+}
+
+// Over a link that resets every few hundred bytes, a download keeps what it
+// has and asks for the rest, so a file larger than any one connection
+// survives still arrives.
+func TestRestoreToResumesAFileAcrossResets(t *testing.T) {
+ src := t.TempDir()
+ dbdoortest.Replicate(t, src, 2)
+ var gets atomic.Int32
+ got, err := RestoreTo(context.Background(), NewClient(flaky(newTestDoor(t, src), 100, &gets)), t.TempDir())
+ if err != nil {
+ t.Fatalf("RestoreTo over a resetting link = %v, want it to resume and finish", err)
+ }
+ defer os.RemoveAll(got.Dir)
+ if rows := countRows(t, got.Path); rows != 2*dbdoortest.Rows {
+ t.Fatalf("restored %d rows, want %d", rows, 2*dbdoortest.Rows)
+ }
+}
+
+// A link that delivers nothing is given up on after a bounded number of
+// attempts, not retried forever.
+func TestRestoreToGivesUpOnALinkThatMakesNoProgress(t *testing.T) {
+ src := t.TempDir()
+ dbdoortest.Replicate(t, src, 1)
+ var gets atomic.Int32
+ parent := t.TempDir()
+ if _, err := RestoreTo(context.Background(), NewClient(flaky(newTestDoor(t, src), 0, &gets)), parent); err == nil {
+ t.Fatal("RestoreTo over a dead link succeeded")
+ }
+ if n := gets.Load(); n > 40 {
+ t.Fatalf("made %d attempts at a link that never delivered a byte", n)
+ }
+ assertEmptyDir(t, parent)
+}
+
+// Ctrl-C after the last file has arrived still stops the restore. The
+// local half (litestream's compaction over the downloaded files) does not
+// check the context itself, so the downloaded files do, and RestoreTo
+// checks again before CheckDB. Nothing is handed over, nothing left.
+func TestRestoreToStopsWhenCancelledAfterTheDownloads(t *testing.T) {
+ src := t.TempDir()
+ dbdoortest.Replicate(t, src, 3)
+ door := newTestDoor(t, src)
+ files := len(collect(t, NewClient(door.do), 0, 0))
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ var gets atomic.Int32
+ do := func(c context.Context, tail string, h http.Header) (*http.Response, error) {
+ res, err := door.do(c, tail, h)
+ if err == nil && strings.Contains(strings.TrimPrefix(tail, "/ltx/"), "/") && int(gets.Add(1)) == files {
+ cancel() // the last download is in hand; only local work is left
+ }
+ return res, err
+ }
+ parent := t.TempDir()
+ if _, err := RestoreTo(ctx, NewClient(do), parent); !errors.Is(err, context.Canceled) {
+ t.Fatalf("RestoreTo cancelled after its downloads = %v, want context.Canceled", err)
+ }
+ assertEmptyDir(t, parent)
+}
+
+// answer is a canned door response.
+func answer(status int, retryAfter string) *http.Response {
+ rec := httptest.NewRecorder()
+ if retryAfter != "" {
+ rec.Header().Set("Retry-After", retryAfter)
+ }
+ rec.WriteHeader(status)
+ io.WriteString(rec, `{"error":"busy"}`)
+ return rec.Result()
+}
+
+// rangeStart is the offset a request's Range header asks for.
+func rangeStart(h http.Header) int64 {
+ off, _, _ := ParseRange(h.Get("Range"))
+ return off
+}
+
+// A reopen of the rest of a file that meets a busy door (503, or 429 with
+// Retry-After) is retried in place, keeping the bytes already downloaded,
+// rather than thrown back to litestream, which would start the file again.
+func TestRestoreToRetriesABusyReopenWithoutLosingProgress(t *testing.T) {
+ defer func(old time.Duration) { dbDownloadBackoff = old }(dbDownloadBackoff)
+ dbDownloadBackoff = time.Millisecond
+ src := t.TempDir()
+ dbdoortest.Replicate(t, src, 2)
+ door := newTestDoor(t, src)
+ var mu sync.Mutex
+ busy := map[string]bool{}
+ var restarts atomic.Int32
+ reached := map[string]int64{}
+ do := func(ctx context.Context, tail string, h http.Header) (*http.Response, error) {
+ if !strings.Contains(strings.TrimPrefix(tail, "/ltx/"), "/") {
+ return door.do(ctx, tail, h)
+ }
+ off := rangeStart(h)
+ key := tail + "@" + strconv.FormatInt(off, 10)
+ mu.Lock()
+ if off < reached[tail] {
+ restarts.Add(1)
+ }
+ reached[tail] = max(reached[tail], off)
+ first := !busy[key]
+ busy[key] = true
+ mu.Unlock()
+ if off > 0 && first {
+ return answer(http.StatusServiceUnavailable, ""), nil
+ }
+ res, err := door.do(ctx, tail, h)
+ if err == nil {
+ res.Body = &flakyBody{ReadCloser: res.Body, left: 100}
+ }
+ return res, err
+ }
+ got, err := RestoreTo(context.Background(), NewClient(do), t.TempDir())
+ if err != nil {
+ t.Fatalf("RestoreTo past busy reopens = %v, want it to retry in place and finish", err)
+ }
+ defer os.RemoveAll(got.Dir)
+ if n := restarts.Load(); n != 0 {
+ t.Fatalf("%d requests went back to an earlier offset: progress was thrown away", n)
+ }
+ if rows := countRows(t, got.Path); rows != 2*dbdoortest.Rows {
+ t.Fatalf("restored %d rows, want %d", rows, 2*dbdoortest.Rows)
+ }
+}
+
+// A 429's Retry-After is honoured before the next attempt.
+func TestRestoreToHonoursRetryAfter(t *testing.T) {
+ defer func(old time.Duration) { dbDownloadBackoff = old }(dbDownloadBackoff)
+ dbDownloadBackoff = time.Millisecond
+ src := t.TempDir()
+ dbdoortest.Replicate(t, src, 1)
+ door := newTestDoor(t, src)
+ var mu sync.Mutex
+ var refusedAt, nextAt time.Time
+ do := func(ctx context.Context, tail string, h http.Header) (*http.Response, error) {
+ if !strings.Contains(strings.TrimPrefix(tail, "/ltx/"), "/") {
+ return door.do(ctx, tail, h)
+ }
+ mu.Lock()
+ defer mu.Unlock()
+ if rangeStart(h) > 0 && refusedAt.IsZero() {
+ refusedAt = time.Now()
+ return answer(http.StatusTooManyRequests, "1"), nil
+ }
+ if !refusedAt.IsZero() && nextAt.IsZero() {
+ nextAt = time.Now()
+ }
+ res, err := door.do(ctx, tail, h)
+ if err == nil && rangeStart(h) == 0 {
+ res.Body = &flakyBody{ReadCloser: res.Body, left: 100}
+ }
+ return res, err
+ }
+ got, err := RestoreTo(context.Background(), NewClient(do), t.TempDir())
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer os.RemoveAll(got.Dir)
+ if refusedAt.IsZero() || nextAt.Sub(refusedAt) < 900*time.Millisecond {
+ t.Fatalf("retried %v after a 429 with Retry-After: 1", nextAt.Sub(refusedAt))
+ }
+}
+
+// The wait between attempts ends the moment the person stops.
+func TestRestoreToBackoffIsCancellable(t *testing.T) {
+ defer func(old time.Duration) { dbDownloadBackoff = old }(dbDownloadBackoff)
+ dbDownloadBackoff = time.Hour
+ src := t.TempDir()
+ dbdoortest.Replicate(t, src, 1)
+ door := newTestDoor(t, src)
+ do := func(ctx context.Context, tail string, h http.Header) (*http.Response, error) {
+ if strings.Contains(strings.TrimPrefix(tail, "/ltx/"), "/") && rangeStart(h) > 0 {
+ return answer(http.StatusServiceUnavailable, ""), nil
+ }
+ res, err := door.do(ctx, tail, h)
+ if err == nil && strings.Contains(strings.TrimPrefix(tail, "/ltx/"), "/") {
+ res.Body = &flakyBody{ReadCloser: res.Body, left: 100}
+ }
+ return res, err
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ time.AfterFunc(200*time.Millisecond, cancel)
+ start := time.Now()
+ parent := t.TempDir()
+ if _, err := RestoreTo(ctx, NewClient(do), parent); !errors.Is(err, context.Canceled) {
+ t.Fatalf("RestoreTo cancelled during a backoff = %v, want context.Canceled", err)
+ }
+ if d := time.Since(start); d > 5*time.Second {
+ t.Fatalf("the stop took %v to land", d)
+ }
+ assertEmptyDir(t, parent)
+}
+
+// A door that stays busy is given up on: only attempts that deliver no
+// byte count, and there are a bounded number of them.
+func TestRestoreToGivesUpOnAPermanentlyBusyDoor(t *testing.T) {
+ defer func(old time.Duration) { dbDownloadBackoff = old }(dbDownloadBackoff)
+ dbDownloadBackoff = time.Millisecond
+ src := t.TempDir()
+ dbdoortest.Replicate(t, src, 1)
+ door := newTestDoor(t, src)
+ var reopens atomic.Int32
+ do := func(ctx context.Context, tail string, h http.Header) (*http.Response, error) {
+ if strings.Contains(strings.TrimPrefix(tail, "/ltx/"), "/") && rangeStart(h) > 0 {
+ reopens.Add(1)
+ return answer(http.StatusServiceUnavailable, ""), nil
+ }
+ res, err := door.do(ctx, tail, h)
+ if err == nil && strings.Contains(strings.TrimPrefix(tail, "/ltx/"), "/") {
+ res.Body = &flakyBody{ReadCloser: res.Body, left: 100}
+ }
+ return res, err
+ }
+ parent := t.TempDir()
+ if _, err := RestoreTo(context.Background(), NewClient(do), parent); err == nil {
+ t.Fatal("RestoreTo against a door that never recovers succeeded")
+ }
+ if n := reopens.Load(); n == 0 || n > 100 {
+ t.Fatalf("%d reopens against a permanently busy door", n)
+ }
+ assertEmptyDir(t, parent)
+}
+
+// A stop that lands while the copy is being checked still withdraws it:
+// RestoreTo checks the context after CheckDB and hands nothing over.
+func TestRestoreToStopsWhenCancelledDuringTheCheck(t *testing.T) {
+ src := t.TempDir()
+ dbdoortest.Replicate(t, src, 1)
+ ctx, cancel := context.WithCancel(context.Background())
+ defer cancel()
+ defer func(old func(context.Context, string) error) { checkDB = old }(checkDB)
+ var sawCtx bool
+ checkDB = func(c context.Context, path string) error {
+ sawCtx = c == ctx
+ cancel() // the stop arrives as the check finishes
+ return nil
+ }
+ parent := t.TempDir()
+ if _, err := RestoreTo(ctx, NewClient(newTestDoor(t, src).do), parent); !errors.Is(err, context.Canceled) {
+ t.Fatalf("RestoreTo cancelled during the check = %v, want context.Canceled", err)
+ }
+ if !sawCtx {
+ t.Fatal("the check did not get the restore's context")
+ }
+ assertEmptyDir(t, parent)
+}
+
+// The FIRST open of a file meeting a full door (429) waits and asks again,
+// the same as a reopen does, instead of failing straight back to
+// litestream, whose own retries ignore Retry-After.
+func TestRestoreToRetriesABusyFirstOpen(t *testing.T) {
+ defer func(old time.Duration) { dbDownloadBackoff = old }(dbDownloadBackoff)
+ dbDownloadBackoff = time.Millisecond
+ src := t.TempDir()
+ dbdoortest.Replicate(t, src, 2)
+ door := newTestDoor(t, src)
+ var mu sync.Mutex
+ refused := map[string]int{}
+ do := func(ctx context.Context, tail string, h http.Header) (*http.Response, error) {
+ if !strings.Contains(strings.TrimPrefix(tail, "/ltx/"), "/") {
+ return door.do(ctx, tail, h)
+ }
+ mu.Lock()
+ n := refused[tail]
+ refused[tail] = n + 1
+ mu.Unlock()
+ if rangeStart(h) == 0 && n < 4 {
+ return answer(http.StatusTooManyRequests, ""), nil
+ }
+ return door.do(ctx, tail, h)
+ }
+ got, err := RestoreTo(context.Background(), NewClient(do), t.TempDir())
+ if err != nil {
+ t.Fatalf("RestoreTo past four busy first opens per file = %v, want it to wait and finish", err)
+ }
+ defer os.RemoveAll(got.Dir)
+ if rows := countRows(t, got.Path); rows != 2*dbdoortest.Rows {
+ t.Fatalf("restored %d rows, want %d", rows, 2*dbdoortest.Rows)
+ }
+}
diff --git a/internal/lease/check.go b/internal/lease/check.go
index 8fdac6469..28a516964 100644
--- a/internal/lease/check.go
+++ b/internal/lease/check.go
@@ -33,6 +33,13 @@ var ErrNoReplica = errors.New("no replica in prefix")
// database passes integrity_check and would be useless in the recovery it was
// meant for.
func CheckDB(path string) error {
+ return CheckDBContext(context.Background(), path)
+}
+
+// CheckDBContext is CheckDB that stops when ctx does: the integrity scan
+// reads every page, and a caller that has been asked to stop (carlos db,
+// on Ctrl-C) should not finish a full scan first.
+func CheckDBContext(ctx context.Context, path string) error {
db, err := sql.Open("sqlite", "file:"+path+"?mode=ro")
if err != nil {
return err
@@ -41,19 +48,19 @@ func CheckDB(path string) error {
db.SetMaxOpenConns(1)
// busy_timeout first, always (Global Constraints) — and on a garbage file
// this first statement is what fails loudly instead of a later query.
- if _, err := db.Exec("PRAGMA busy_timeout=5000"); err != nil {
- return fmt.Errorf("open: %v", err)
+ if _, err := db.ExecContext(ctx, "PRAGMA busy_timeout=5000"); err != nil {
+ return fmt.Errorf("open: %w", err)
}
var res string
- if err := db.QueryRow("PRAGMA integrity_check").Scan(&res); err != nil {
- return fmt.Errorf("integrity_check: %v", err)
+ if err := db.QueryRowContext(ctx, "PRAGMA integrity_check").Scan(&res); err != nil {
+ return fmt.Errorf("integrity_check: %w", err)
}
if res != "ok" {
return fmt.Errorf("integrity_check: %s", res)
}
var tables int
- if err := db.QueryRow("SELECT count(*) FROM sqlite_master WHERE type='table'").Scan(&tables); err != nil {
- return fmt.Errorf("schema count: %v", err)
+ if err := db.QueryRowContext(ctx, "SELECT count(*) FROM sqlite_master WHERE type='table'").Scan(&tables); err != nil {
+ return fmt.Errorf("schema count: %w", err)
}
if tables == 0 {
return errors.New("restored database has no tables")
diff --git a/internal/lease/check_test.go b/internal/lease/check_test.go
index 65828388d..8156b30e5 100644
--- a/internal/lease/check_test.go
+++ b/internal/lease/check_test.go
@@ -221,3 +221,18 @@ func TestVerifyReplicaReturnsTheRestoredChangeCounter(t *testing.T) {
t.Fatalf("restored change counter = %d, want %d", got, want)
}
}
+
+// A stop reaches the integrity scan: CheckDBContext returns the context's
+// error instead of reading the whole file first.
+func TestCheckDBContextHonoursCancellation(t *testing.T) {
+ path := filepath.Join(t.TempDir(), "c.db")
+ seedDB(t, path)
+ if err := CheckDBContext(context.Background(), path); err != nil {
+ t.Fatalf("live context: %v", err)
+ }
+ ctx, cancel := context.WithCancel(context.Background())
+ cancel()
+ if err := CheckDBContext(ctx, path); !errors.Is(err, context.Canceled) {
+ t.Fatalf("cancelled context = %v, want context.Canceled", err)
+ }
+}
diff --git a/internal/lease/endpoint.go b/internal/lease/endpoint.go
index c945f0dd1..d01061b8c 100644
--- a/internal/lease/endpoint.go
+++ b/internal/lease/endpoint.go
@@ -128,3 +128,22 @@ func newReplicaClient(bucket, prefix, region string) *lss3.ReplicaClient {
}
return c
}
+
+// ReadReplicaClient is newReplicaClient for READERS outside this package —
+// the console's db door (spec 2026-09-02 §6). Identical construction on
+// purpose, region resolution included (Restore's resolveRegion): a second
+// hand-rolled client is how the endpoint override, the path-style flag and
+// the signing region drift apart between the writer and the reader of the
+// same bucket.
+//
+// It returns the same type the writer uses, so nothing here can enforce
+// read-only by construction; that guarantee lives in internal/dbdoor's
+// client, which has no write methods at all, and in the door only ever
+// calling LTXFiles and OpenLTXFile on what this returns.
+func ReadReplicaClient(ctx context.Context, bucket, prefix, region string) (*lss3.ReplicaClient, error) {
+ region, err := resolveRegion(ctx, region)
+ if err != nil {
+ return nil, err
+ }
+ return newReplicaClient(bucket, prefix, region), nil
+}
diff --git a/internal/lease/endpoint_test.go b/internal/lease/endpoint_test.go
index 75fb3f9f1..68346f032 100644
--- a/internal/lease/endpoint_test.go
+++ b/internal/lease/endpoint_test.go
@@ -301,3 +301,40 @@ func TestRestoreCreatesTheOutputPathsParentDirectory(t *testing.T) {
t.Fatalf("Restore did not create the output path's parent directory: %v", err)
}
}
+
+// ReadReplicaClient is the db door's reader (spec 2026-09-02 §6). It must
+// carry the coordinates it was given and sign for the region lease.Restore
+// would: an explicit region wins, and an empty one falls to the same chain
+// resolution the writer uses — the endpoint override's non-AWS default when
+// nothing else answers.
+func TestReadReplicaClientCarriesBucketPrefixRegion(t *testing.T) {
+ fakeCreds(t)
+ c, err := ReadReplicaClient(context.Background(), "data-bucket", "instances/a.example.com", "eu-west-1")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if c.Bucket != "data-bucket" {
+ t.Errorf("Bucket = %q, want data-bucket", c.Bucket)
+ }
+ if c.Path != "instances/a.example.com" {
+ t.Errorf("Path = %q, want instances/a.example.com", c.Path)
+ }
+ if c.Region != "eu-west-1" {
+ t.Errorf("Region = %q, want eu-west-1", c.Region)
+ }
+}
+
+func TestReadReplicaClientResolvesAnEmptyRegionLikeRestore(t *testing.T) {
+ fakeCreds(t)
+ t.Setenv("CARLOS_S3_ENDPOINT", "http://127.0.0.1:1")
+ c, err := ReadReplicaClient(context.Background(), "data-bucket", "instances/a.example.com", "")
+ if err != nil {
+ t.Fatal(err)
+ }
+ if c.Region != lss3.DefaultRegion {
+ t.Errorf("Region = %q, want litestream's non-AWS default %q (the writer's resolution)", c.Region, lss3.DefaultRegion)
+ }
+ if c.Endpoint != "http://127.0.0.1:1" || !c.ForcePathStyle {
+ t.Errorf("Endpoint = %q ForcePathStyle = %v, want the override applied", c.Endpoint, c.ForcePathStyle)
+ }
+}
diff --git a/main.go b/main.go
index b3b52dff7..16f2ccd5b 100644
--- a/main.go
+++ b/main.go
@@ -73,6 +73,8 @@ func main() {
runDeploy(args)
case "restart":
runRestart(args)
+ case "db":
+ runDB(args)
case "steering":
runSteering(args)
case "schedule":
@@ -197,6 +199,8 @@ Your apps — ship and run software on a CARLOS deployment
nothing the app already serves changes: deploy | ls | rm
restart cycle an app's instances — no new version, no config change;
--host or --environment narrows it, --dry-run names the set
+ db read a copy of an instance's database, restored from its
+ replica: pull (save it) | query (one read-only query)
schedule recurring or one-shot work per app: ls | set | rm | run
store object storage: create | status | apply | approvals | approve | decline | grant | rotate
relay the platform STUN/TURN relay: serve (what the unit runs)