amadan

carlos / platform Public

console: carlos instances create --from, reserve, restore, upload, publish (Plan 5 Task 5)

createFrom takes over from the create door once every ordinary check has
passed. It checks placement, the data bucket and the seed floor, resolves
the source, plans the restore and admits its exact size, and reserves
scratch. It then checks the store, reserves the record, provisions the
seed's own store, and restores at the planned TXID under a 30-minute
deadline and a size watcher. Last, it uploads the snapshot, re-checks the
sibling apex and publishes. Every failure after the reservation aborts the
record. The answer is held until scratch is removed and the reservation
released.

The watcher measures replica.db.tmp as well as replica.db (litestream
renames only at the end), and a finished restore larger than planned is
refused too. The console wires SeedData from the db door's data bucket
and SeedScratchDir beside CONSOLE_DB.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 872b3b17a5c3238af47a146a972b11155b60ae61 parent 941e6db
10 files changed, +1283 −8
  • approved_seeded_copy_test.go +4 −2
  • cmd/console/main.go +14 −0
  • internal/console/apiinstances.go +25 −0
  • internal/console/apiinstanceseed.go +494 −0
  • internal/console/apiinstanceseed_test.go +721 −0
  • internal/console/claim_authority_census_test.go +1 −1
  • internal/console/seedfloor.go +8 −1
  • internal/console/seedscratch.go +7 −1
  • internal/console/server.go +7 −3
  • internal/console/testdata/claim-authority-sites.txt +2 −0
diff --git a/approved_seeded_copy_test.go b/approved_seeded_copy_test.go
index fcaa131aa..80f1fd205 100644
--- a/approved_seeded_copy_test.go
+++ b/approved_seeded_copy_test.go
@@ -33,8 +33,10 @@ var seededCopyIDs = func() []string {
// seededWiredIDs grows as the plans wire rows. Plan 1 wires none; Plan 3
// Task 3 wires SD-N1, Task 5 SD-N2 to SD-N8 and SD-N113, Task 6 SD-N9 to SD-N13,
-// SD-N111 and SD-N112; Plan 4 Task 3 SD-N14 to SD-N19; Plan 5 Task 3 SD-N27 to SD-N29.
-var seededWiredIDs = []string{"SD-N1", "SD-N2", "SD-N3", "SD-N4", "SD-N5", "SD-N6", "SD-N7", "SD-N8", "SD-N113", "SD-N9", "SD-N10", "SD-N11", "SD-N12", "SD-N13", "SD-N111", "SD-N112", "SD-N14", "SD-N15", "SD-N16", "SD-N17", "SD-N18", "SD-N19", "SD-N27", "SD-N28", "SD-N29"}
+// SD-N111 and SD-N112; Plan 4 Task 3 SD-N14 to SD-N19; Plan 5 Task 3 SD-N27 to SD-N29,
+// Task 5 SD-N20 to SD-N26, SD-N30 to SD-N39 and SD-N41.
+var seededWiredIDs = []string{"SD-N1", "SD-N2", "SD-N3", "SD-N4", "SD-N5", "SD-N6", "SD-N7", "SD-N8", "SD-N113", "SD-N9", "SD-N10", "SD-N11", "SD-N12", "SD-N13", "SD-N111", "SD-N112", "SD-N14", "SD-N15", "SD-N16", "SD-N17", "SD-N18", "SD-N19", "SD-N27", "SD-N28", "SD-N29",
+ "SD-N20", "SD-N21", "SD-N22", "SD-N23", "SD-N24", "SD-N25", "SD-N26", "SD-N30", "SD-N31", "SD-N32", "SD-N33", "SD-N34", "SD-N35", "SD-N36", "SD-N37", "SD-N38", "SD-N39", "SD-N41"}
func mustParseSeededRecord(t *testing.T) []copyRow {
t.Helper()
diff --git a/cmd/console/main.go b/cmd/console/main.go
index 46635fbf4..4be3a03f7 100644
--- a/cmd/console/main.go
+++ b/cmd/console/main.go
@@ -299,6 +299,16 @@ func serve() {
log.Fatalf("console: versioned database reader: %v", err)
}
}
+ // The seed doors write seeds/ at the root of the same data bucket the db
+ // door reads and the box's activator writes (seeded-instances spec §3.1).
+ var seedData release.Store
+ if dbBucket != "" {
+ if sd, err := release.NewS3StoreIn(context.Background(), dbBucket, dbRegion); err != nil {
+ log.Printf("console: seed data store: %v (the seed doors will answer 503)", err)
+ } else {
+ seedData = sd
+ }
+ }
// Before the server exists: every door that seals reads this switch.
appconfig.SetBoundSealing(secretsEnvelopeV2())
geoReader := consoleGeoReader(strings.TrimSpace(os.Getenv("CONSOLE_GEOIP_DB")))
@@ -324,6 +334,10 @@ func serve() {
// Next to CONSOLE_DB, on disk (an upload can be 1 GiB), and used
// for nothing else: NewServer clears it (geoip spec §4.2).
GeoipStagingDir: filepath.Join(filepath.Dir(envOr("CONSOLE_DB", "/data/console/console.db")), "geoip-staging"),
+ // --from restores and import spools, next to CONSOLE_DB on disk;
+ // NewServer clears the pool's own leftovers from it.
+ SeedScratchDir: filepath.Join(filepath.Dir(envOr("CONSOLE_DB", "/data/console/console.db")), "scratch", "seeds"),
+ SeedData: seedData,
// The feed (geoip spec §10.1). The key goes nowhere but MaxMind.
GeoipFeed: os.Getenv("CONSOLE_GEOIP_FEED"),
GeoipMaxMindAccount: strings.TrimSpace(os.Getenv("CONSOLE_GEOIP_MAXMIND_ACCOUNT")),
diff --git a/internal/console/apiinstances.go b/internal/console/apiinstances.go
index 2f781667e..ce81107e2 100644
--- a/internal/console/apiinstances.go
+++ b/internal/console/apiinstances.go
@@ -696,6 +696,10 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request)
// kind/environment's — one flat body, no nesting for the caller to
// discover.
replicaFields
+ // The seed doors (seeded-instances spec §4, §5.1): a create that
+ // starts from a sibling's replica or an uploaded snapshot. Both run
+ // every check below, then leave this handler (apiinstanceseed.go).
+ From *seedFromReq `json:"from,omitempty"`
}
if !readJSON(w, r, &req) {
return
@@ -704,6 +708,13 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request)
if !ok {
return
}
+ seeded := req.From != nil
+ if seeded {
+ if ref := seedCommonRefusal(strings.TrimSpace(req.Kind), req.Addrs, strings.TrimSpace(req.Role), req.Replicas); ref != nil {
+ jsonError(w, ref.Status, ref.Msg)
+ return
+ }
+ }
host := strings.TrimSpace(req.Host)
// Path-traversal defence stated as such (spec §11): this host becomes a
// bucket key segment here and, on the box, the <host>.sock and
@@ -1093,6 +1104,13 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request)
// Stamped BEFORE ValidateShape so the shape rail sees the same
// environment the box will, and before the ceiling and the walk below so
// neither can be asked about a record that was never legal.
+ //
+ // A seeded instance takes no explicit environment under the policy: its
+ // store must be new and empty, and a named one may be shared (spec §4).
+ if seeded && m.PrivateStorage && strings.TrimSpace(req.Environment) != "" {
+ s.refuseSeededEnvironment(w)
+ return
+ }
if m.PrivateStorage {
env, derived := rec.Environment, false
if env == "" {
@@ -1167,6 +1185,13 @@ func (s *Server) handleAPIInstanceCreate(w http.ResponseWriter, r *http.Request)
jsonError(w, ref.Status, ref.Msg)
return
}
+ // Every ordinary check has passed: a seed door takes over from here, and
+ // writes its own record (apiinstanceseed.go).
+ if req.From != nil {
+ s.createFrom(w, r, seedCreate{acct: acct, app: app, principal: principal, message: message, m: m,
+ rec: rec, placed: placed, regionFrom: regionFrom, siblingLabel: siblingLabel}, *req.From)
+ return
+ }
// A fresh budget for the write and its rollback: validation and the
// rail's listings may have spent most of the original one.
cancel()
diff --git a/internal/console/apiinstanceseed.go b/internal/console/apiinstanceseed.go
new file mode 100644
index 000000000..a01e92d1d
--- /dev/null
+++ b/internal/console/apiinstanceseed.go
@@ -0,0 +1,494 @@
+// The --from seed door (seeded-instances spec §4): `carlos instances create
+// --from <host>` creates an instance whose database starts as a copy of a
+// sibling's replica. handleAPIInstanceCreate runs every ordinary check,
+// then hands over here.
+
+package console
+
+import (
+ "bytes"
+ "context"
+ "crypto/sha256"
+ "encoding/hex"
+ "errors"
+ "fmt"
+ "io"
+ "io/fs"
+ "log"
+ "net/http"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "sync/atomic"
+ "time"
+
+ "github.com/carlosframework/platform/internal/dbdoor"
+ "github.com/carlosframework/platform/internal/objstore"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/registry"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/seeds"
+)
+
+// Seams for the tests.
+var (
+ seedMaxBytes = seeds.MaxBytes
+ planRestore = dbdoor.PlanRestore
+ restoreToTarget = dbdoor.RestoreToTarget
+ // seedRestoreDeadline bounds the restore and its check (spec §4 step 5).
+ seedRestoreDeadline = 30 * time.Minute
+ // restoreWatchEvery is how often the size watcher looks (spec §4 step 5).
+ restoreWatchEvery = time.Second
+)
+
+// seedUploadDeadline bounds the snapshot's one PUT, after the restore's own
+// deadline: up to 4 GiB to the region's own bucket. The record is reserving
+// for the whole restore, store walk and upload, and the sweep aborts a
+// --from that has been reserving for an hour (spec §3.1), so the three
+// together stay well under that.
+const seedUploadDeadline = 20 * time.Minute
+
+// seedReadSlack is how far past a step's own deadline the connection's read
+// deadline is set, so the step's deadline is always the one that fires.
+const seedReadSlack = 5 * time.Minute
+
+// seedFromReq is the create body's "from" (spec §4). TXID 0 is the latest
+// copy; the CLI refuses a non-positive --at.
+type seedFromReq struct {
+ Host string `json:"host"`
+ TXID uint64 `json:"txid,omitempty"`
+}
+
+// seedCreate is what handleAPIInstanceCreate has decided by the time it
+// hands a seed door over: every ordinary check has passed. siblingLabel is
+// the create door's own (set when the host sits at another label of the
+// account's platform namespace), for its re-check before publishing.
+type seedCreate struct {
+ acct *Account
+ app, principal, message string
+ m *provision.Manifest
+ rec provision.Record
+ placed, regionFrom string
+ siblingLabel string
+}
+
+// seedCommonRefusal is both seed doors' shape rule: an ordinary instance.
+func seedCommonRefusal(kind string, addrs []string, role string, replicas []string) *doorRefusal {
+ if (kind != "" && kind != "instance") || len(addrs) > 0 || role != "" || len(replicas) > 0 {
+ return &doorRefusal{http.StatusBadRequest, "a seeded instance is an ordinary instance. Drop --kind, --addr and --role."}
+ }
+ return nil
+}
+
+// seedPlacementRefusal is both seed doors' placement rule: the platform
+// keeps a replica only of an instance it hosts itself.
+func seedPlacementRefusal(c seedCreate) *doorRefusal {
+ if c.placed != "" || c.m.Pool != "" {
+ return &doorRefusal{http.StatusConflict, fmt.Sprintf("%s places its instances in a pool or on a fleet, so it cannot take a seeded instance.", c.app)}
+ }
+ return nil
+}
+
+// refuseSeededEnvironment is both seed doors' answer to an explicit
+// --environment on an app with private storage (spec §4 Storage).
+func (s *Server) refuseSeededEnvironment(w http.ResponseWriter) {
+ jsonError(w, http.StatusConflict, "a seeded instance gets its own environment and an empty store")
+}
+
+// seedStoreTakenRefusal names the store a seeded create may not use.
+func seedStoreTakenRefusal(env, bucket string) *doorRefusal {
+ return &doorRefusal{http.StatusConflict, fmt.Sprintf("environment %s already has a store (%s). A seeded instance needs a new, empty one.", env, bucket)}
+}
+
+// seedStoreRefusal is spec §4 Storage's existing-store rule for both doors,
+// checked before the record is reserved. instanceStoreProvisionFor checks
+// it again under the store lock. failed is the door's own retry sentence,
+// answered with a 502 when the store record cannot be read.
+func (s *Server) seedStoreRefusal(ctx context.Context, c seedCreate, seedID, failed string) *doorRefusal {
+ if !c.m.PrivateStorage {
+ return nil
+ }
+ rec, _, err := objstore.GetIn(ctx, s.cfg.Releases, c.acct.Sqid, c.app, c.rec.Environment)
+ if err != nil {
+ log.Printf("console: seed store check %s/%s/%s: %v", c.acct.Sqid, c.app, c.rec.Environment, err)
+ return &doorRefusal{http.StatusBadGateway, failed}
+ }
+ if !seedOwnsStore(rec, seedID) {
+ return seedStoreTakenRefusal(c.rec.Environment, storeBucketName(rec))
+ }
+ return nil
+}
+
+func seedGiB(n int64) string { return fmt.Sprintf("%.1f GiB", float64(n)/(1<<30)) }
+
+// createFrom is --from (spec §4). It answers once, after its scratch is
+// removed and its reservation released, so a caller that retries at once
+// never meets its own leftovers.
+//
+// The request runs as long as the copy does. The connection's read
+// deadline is set past each step's own deadline (seedReadSlack), so the
+// server's ReadTimeout is never the one that ends a copy. Go 1.26's
+// net/http already clears that deadline once the body is read; this keeps
+// the door's deadlines in charge whatever the server does. A client that
+// goes away still cancels the request's context.
+func (s *Server) createFrom(w http.ResponseWriter, r *http.Request, c seedCreate, from seedFromReq) {
+ rc := http.NewResponseController(w)
+ extend := func(d time.Duration) {
+ if err := rc.SetReadDeadline(time.Now().Add(d + seedReadSlack)); err != nil && !errors.Is(err, http.ErrNotSupported) {
+ log.Printf("console: from %s: extend the read deadline: %v", c.rec.Host, err)
+ }
+ }
+ extend(seedRestoreDeadline)
+ defer func() { _ = rc.SetReadDeadline(time.Time{}) }()
+ held := &heldResponse{header: w.Header()}
+ s.copyFrom(held, r, c, from, func() { extend(seedUploadDeadline) })
+ held.flush(w)
+}
+
+func (s *Server) copyFrom(w http.ResponseWriter, r *http.Request, c seedCreate, from seedFromReq, beforeUpload func()) {
+ ctx := r.Context()
+ acct, app, host := c.acct, c.app, c.rec.Host
+ source := strings.TrimSpace(from.Host)
+ failed := fmt.Sprintf("could not copy %s. Nothing was created. Try again.", source)
+ if ref := seedPlacementRefusal(c); ref != nil {
+ jsonError(w, ref.Status, ref.Msg)
+ return
+ }
+ if s.cfg.SeedData == nil || s.cfg.HibernateBucket == "" {
+ jsonError(w, http.StatusServiceUnavailable, "this console has no data bucket set up, so it cannot create a seeded instance.")
+ return
+ }
+ checks, cancel := context.WithTimeout(ctx, instanceStoreTimeout)
+ defer cancel()
+ if ref := s.seedRegionReady(checks, c.rec.Region, false); ref != nil {
+ jsonError(w, ref.Status, ref.Msg)
+ return
+ }
+ src, err := s.resolveDBSource(checks, acct, c.principal, app, source)
+ if ref := s.fromSourceRefusal(src, source, app, err); ref != nil {
+ jsonError(w, ref.Status, ref.Msg)
+ return
+ }
+ if src.rec.Region != c.rec.Region {
+ jsonError(w, http.StatusConflict, "a copy stays in its source's region. To run it elsewhere, create it here and then run carlos move.")
+ return
+ }
+ rd, err := s.dbReplica(ctx, src.prefix)
+ if err != nil {
+ log.Printf("console: from %s: replica client: %v", source, err)
+ jsonError(w, http.StatusBadGateway, failed)
+ return
+ }
+ client := dbdoor.NewClient(replicaDoer(rd))
+ planCtx, planCancel := context.WithTimeout(ctx, instanceStoreTimeout)
+ plan, err := planRestore(planCtx, client, from.TXID)
+ planCancel()
+ switch {
+ case from.TXID != 0 && (errors.Is(err, dbdoor.ErrHistoryGone) || errors.Is(err, dbdoor.ErrNothingReplicated)):
+ jsonError(w, http.StatusConflict, fmt.Sprintf("history before %d is no longer kept", from.TXID))
+ return
+ case errors.Is(err, dbdoor.ErrNothingReplicated):
+ jsonError(w, http.StatusConflict, fmt.Sprintf("%s has not replicated anything yet. Wake it once, then try again.", source))
+ return
+ case err != nil:
+ // A file that vanished under the plan, a header that does not
+ // describe its file, a bucket read: a retry plans again.
+ log.Printf("console: from %s: plan: %v", source, err)
+ jsonError(w, http.StatusBadGateway, failed)
+ return
+ }
+ if plan.RestoredBytes > seedMaxBytes {
+ jsonError(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("a copy of %s would be %s, over the 4 GiB limit.", source, seedGiB(plan.RestoredBytes)))
+ return
+ }
+ releaseScratch, ref := s.seedScratch.reserve(acct.Sqid+"/"+app, plan.RestoredBytes+plan.LTXBytes)
+ if ref != nil {
+ jsonError(w, ref.Status, ref.Msg)
+ return
+ }
+ defer releaseScratch()
+ id, err := seeds.NewID()
+ if err != nil {
+ log.Printf("console: from %s: seed id: %v", source, err)
+ jsonError(w, http.StatusBadGateway, failed)
+ return
+ }
+ storeCtx, storeCancel := context.WithTimeout(ctx, instanceStoreTimeout)
+ ref = s.seedStoreRefusal(storeCtx, c, id, failed)
+ storeCancel()
+ if ref != nil {
+ jsonError(w, ref.Status, ref.Msg)
+ return
+ }
+
+ // Reserve the record (spec §4 step 7.1). It is reserved here, after the
+ // size, scratch and store checks rather than straight after resolving
+ // the source: a refusal before this point leaves nothing behind, where
+ // an earlier record would turn each into an aborted record holding the
+ // host until it is finished (a recorded deviation, Plan 5 M10).
+ rec := c.rec
+ rec.Seed = &provision.SeedSource{ID: id, Kind: seeds.KindReplica, From: source, State: seeds.StateReserving,
+ By: c.principal, At: time.Now().UTC()}
+ writeCtx, writeCancel := context.WithTimeout(ctx, instanceStoreTimeout)
+ err = provision.CreateRecord(writeCtx, s.cfg.Releases, rec)
+ writeCancel()
+ if err != nil {
+ if errors.Is(err, release.ErrCASConflict) {
+ jsonError(w, http.StatusConflict, "instance "+host+" already exists")
+ return
+ }
+ log.Printf("console: from %s: reserve %s: %v", source, host, err)
+ jsonError(w, http.StatusBadGateway, failed)
+ return
+ }
+ // From here every failure moves the record to aborted, for the box and
+ // the console's finish to clean up (spec §4 step 7), under a context the
+ // client's departure cannot cancel. A store this seed provisioned is
+ // left to that finish (Plan 8).
+ abort := func(why string) {
+ bg, cancel := context.WithTimeout(context.WithoutCancel(ctx), instanceStoreTimeout)
+ defer cancel()
+ if _, err := provision.SetRecordSeed(bg, s.cfg.Releases, acct.Sqid, app, host, provision.SeedAbort(id)); err != nil {
+ log.Printf("console: from %s: abort %s (seed %s) after %s: %v", source, host, id, why, err)
+ }
+ s.bumpSerial(bg)
+ }
+
+ privateBucket := ""
+ if c.m.PrivateStorage {
+ // The request context: the walk takes its own storeGrantTimeout.
+ storeRec, ok, err := s.instanceStoreProvisionFor(ctx, w, acct, app, rec.Environment, c.principal, id)
+ if !ok {
+ abort("store provisioning")
+ if errors.Is(err, errSeedStoreTaken) {
+ ref := seedStoreTakenRefusal(rec.Environment, storeBucketName(storeRec))
+ jsonError(w, ref.Status, ref.Msg)
+ }
+ return
+ }
+ privateBucket = storeBucketName(storeRec)
+ }
+
+ dir, err := os.MkdirTemp(s.seedScratch.dir(), seedScratchPrefix)
+ if err != nil {
+ log.Printf("console: from %s: scratch: %v", source, err)
+ abort("scratch")
+ ref := seedScratchFull()
+ jsonError(w, ref.Status, ref.Msg)
+ return
+ }
+ defer os.RemoveAll(dir)
+ // Pinned at the planned TXID, never "latest" again: only that restore
+ // produces the size that was admitted, however far the source has moved
+ // on since it was planned.
+ rctx, rcancel := context.WithTimeout(ctx, seedRestoreDeadline)
+ defer rcancel()
+ grew := restoreWatch(rctx, rcancel, dir, plan.RestoredBytes, plan.RestoredBytes+plan.LTXBytes)
+ restored, err := restoreToTarget(rctx, client, dir, plan.TXID)
+ timedOut := errors.Is(rctx.Err(), context.DeadlineExceeded)
+ rcancel() // stops the watcher
+ overgrown := grew()
+ if err == nil && !overgrown {
+ // The watcher's tick can come after a fast restore has finished.
+ st, serr := os.Stat(restored.Path)
+ switch {
+ case serr != nil:
+ err = serr
+ case st.Size() > plan.RestoredBytes:
+ overgrown = true
+ }
+ }
+ if err != nil || overgrown {
+ abort("restore")
+ switch {
+ case overgrown:
+ jsonError(w, http.StatusRequestEntityTooLarge, fmt.Sprintf("the copy of %s grew past its planned size, so it was stopped and no instance was created.", source))
+ case ctx.Err() != nil:
+ // The client went away: nobody to answer.
+ case timedOut:
+ jsonError(w, http.StatusGatewayTimeout, fmt.Sprintf("copying %s took longer than 30 minutes, so no instance was created.", source))
+ case errors.Is(err, dbdoor.ErrCheckFailed):
+ jsonError(w, http.StatusUnprocessableEntity, fmt.Sprintf("the copy of %s failed its integrity check, so no instance was created.", source))
+ case errors.Is(err, dbdoor.ErrHistoryGone) && plan.Target != 0:
+ jsonError(w, http.StatusConflict, fmt.Sprintf("history before %d is no longer kept", from.TXID))
+ default:
+ // Including history lost under a latest copy between its plan
+ // and its restore: a retry plans again.
+ log.Printf("console: from %s: restore: %v", source, err)
+ jsonError(w, http.StatusBadGateway, failed)
+ }
+ return
+ }
+
+ beforeUpload()
+ uctx, ucancel := context.WithTimeout(ctx, seedUploadDeadline)
+ sha, size, err := uploadSnapshot(uctx, s.cfg.SeedData, restored.Path, seeds.SnapshotKey(host, id))
+ ucancel()
+ if err != nil {
+ abort("upload")
+ if ctx.Err() == nil {
+ log.Printf("console: from %s: upload %s: %v", source, host, err)
+ jsonError(w, http.StatusBadGateway, failed)
+ }
+ return
+ }
+ // The create door's re-check after its write (apiinstances.go), made
+ // here before the publish: a reserving record mints nothing, so the
+ // publish is the point of no return.
+ if c.siblingLabel != "" {
+ if live, trashed, err := s.store.AppNameState(acct.ID, c.siblingLabel); err == nil && (live || trashed) {
+ abort("a sibling app took the label")
+ jsonError(w, http.StatusConflict,
+ host+" is app "+c.siblingLabel+"'s platform apex — it was created while this instance was being written; the instance was not kept")
+ return
+ }
+ }
+ pctx, pcancel := context.WithTimeout(ctx, instanceStoreTimeout)
+ defer pcancel()
+ published, err := provision.SetRecordSeed(pctx, s.cfg.Releases, acct.Sqid, app, host,
+ provision.SeedPublishReplica(id, strconv.FormatUint(restored.TXID, 10), sha, size))
+ if err != nil {
+ abort("publish")
+ if ctx.Err() == nil {
+ log.Printf("console: from %s: publish %s: %v", source, host, err)
+ jsonError(w, http.StatusBadGateway, failed)
+ }
+ return
+ }
+ s.auditActivity(acct, app, c.principal, Activity{
+ Action: "instance-seed", TargetType: "instance", Target: host, Message: c.message,
+ Detail: fmt.Sprintf("seeded %s from %s at txid %d", host, source, restored.TXID),
+ })
+ s.bumpSerial(pctx)
+ out := recordView(*published)
+ out.PrivateStoreBucket = privateBucket
+ out.RegionFrom = c.regionFrom
+ out.RegionName = s.placeOf(published.Region, s.candsOrHome(pctx)) // never fleet-placed: refused above
+ writeJSON(w, http.StatusCreated, out)
+}
+
+// fromSourceRefusal maps resolveDBSource's answer to --from's sentences.
+func (s *Server) fromSourceRefusal(src dbSource, source, app string, err error) *doorRefusal {
+ switch {
+ case err == nil:
+ return nil
+ case errors.Is(err, os.ErrNotExist), errors.Is(err, ErrBadHost):
+ return &doorRefusal{http.StatusConflict, fmt.Sprintf("no instance %s in %s. A copy can only come from an instance of the same app in the same account.", source, app)}
+ case errors.Is(err, errSourceClass):
+ return &doorRefusal{http.StatusConflict, fmt.Sprintf("%s is not an instance the platform keeps a replica of, so it cannot be copied.", source)}
+ case errors.Is(err, ErrPooled):
+ return &doorRefusal{http.StatusConflict, fmt.Sprintf("%s runs in the pool %s, so the platform keeps no replica of it to copy.", source, src.rec.Pool)}
+ case errors.Is(err, ErrResidency):
+ if fleet, ok := registry.FleetPlacement(src.rec.Region); ok {
+ return &doorRefusal{http.StatusConflict, fmt.Sprintf("%s runs on your fleet %s, so the platform keeps no replica of it to copy.", source, fleet)}
+ }
+ return &doorRefusal{http.StatusConflict, s.dbResidencyRefusal(src.rec)}
+ case errors.Is(err, errSourceClaim):
+ // Unreachable while seedRegionReady refuses a claim console first.
+ return seedClaimConsoleRefusal()
+ }
+ log.Printf("console: from %s: resolve: %v", source, err)
+ return &doorRefusal{http.StatusBadGateway, fmt.Sprintf("could not copy %s. Nothing was created. Try again.", source)}
+}
+
+// uploadSnapshot hashes path and writes it to key with one create-only
+// PutFrom (spec §4 step 7.2: seekable, under S3's 5 GB single-PUT limit).
+func uploadSnapshot(ctx context.Context, data release.Store, path, key string) (sha string, size int64, err error) {
+ f, err := os.Open(path)
+ if err != nil {
+ return "", 0, err
+ }
+ defer f.Close()
+ h := sha256.New()
+ if size, err = io.Copy(h, f); err != nil {
+ return "", 0, err
+ }
+ if _, err := f.Seek(0, io.SeekStart); err != nil {
+ return "", 0, err
+ }
+ if _, err := release.PutObjectFrom(ctx, data, key, f, size, ""); err != nil {
+ return "", 0, err
+ }
+ return hex.EncodeToString(h.Sum(nil)), size, nil
+}
+
+// restoreWatch is the size watcher (spec §4 step 5). Every
+// restoreWatchEvery it measures the restore under dir and cancels it when
+// the database (litestream's replica.db.tmp while it decodes, replica.db
+// after its rename) grows past dbLimit, or everything the restore holds
+// (the database and its downloaded LTX files) grows past totalLimit, the
+// reservation. grew reports whether it fired.
+func restoreWatch(ctx context.Context, cancel context.CancelFunc, dir string, dbLimit, totalLimit int64) (grew func() bool) {
+ var fired atomic.Bool
+ every := restoreWatchEvery
+ go func() {
+ t := time.NewTicker(every)
+ defer t.Stop()
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-t.C:
+ }
+ if db, total := restoreScratchUse(dir); db > dbLimit || total > totalLimit {
+ fired.Store(true)
+ cancel()
+ return
+ }
+ }
+ }()
+ return fired.Load
+}
+
+// restoreScratchUse measures a restore under dir: the largest database file
+// (replica.db or replica.db.tmp, dbdoor's names) and every regular file
+// together. SQLite's shared-memory index, which the read-only check opens
+// beside the copy, is not the restore's and is not counted.
+func restoreScratchUse(dir string) (db, total int64) {
+ _ = filepath.WalkDir(dir, func(_ string, d fs.DirEntry, err error) error {
+ if err != nil || !d.Type().IsRegular() || strings.HasSuffix(d.Name(), "-shm") {
+ return nil // a file the restore removed between listing and stat is not there
+ }
+ info, err := d.Info()
+ if err != nil {
+ return nil
+ }
+ total += info.Size()
+ if n := d.Name(); n == "replica.db" || n == "replica.db.tmp" {
+ db = max(db, info.Size())
+ }
+ return nil
+ })
+ return db, total
+}
+
+// heldResponse holds a door's answer until its cleanup has run.
+type heldResponse struct {
+ header http.Header
+ status int
+ body bytes.Buffer
+}
+
+func (h *heldResponse) Header() http.Header { return h.header }
+
+func (h *heldResponse) WriteHeader(code int) {
+ if h.status == 0 {
+ h.status = code
+ }
+}
+
+func (h *heldResponse) Write(b []byte) (int, error) {
+ h.WriteHeader(http.StatusOK)
+ return h.body.Write(b)
+}
+
+// flush writes what was held; nothing when the door answered nothing (the
+// client went away).
+func (h *heldResponse) flush(w http.ResponseWriter) {
+ if h.status == 0 {
+ return
+ }
+ w.WriteHeader(h.status)
+ _, _ = w.Write(h.body.Bytes())
+}
diff --git a/internal/console/apiinstanceseed_test.go b/internal/console/apiinstanceseed_test.go
new file mode 100644
index 000000000..18a3a6a6d
--- /dev/null
+++ b/internal/console/apiinstanceseed_test.go
@@ -0,0 +1,721 @@
+package console
+
+import (
+ "context"
+ "crypto/sha256"
+ "database/sql"
+ "encoding/hex"
+ "encoding/json"
+ "errors"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "path/filepath"
+ "strconv"
+ "strings"
+ "testing"
+ "time"
+
+ lsfile "github.com/benbjohnson/litestream/file"
+
+ "github.com/carlosframework/platform/internal/dbdoor"
+ "github.com/carlosframework/platform/internal/dbdoor/dbdoortest"
+ "github.com/carlosframework/platform/internal/objstore"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/seeds"
+ "github.com/carlosframework/platform/internal/svcrole"
+)
+
+const (
+ fromApp = "shop"
+ fromDomain = "shop.example.com"
+ fromSource = "main.shop.example.com"
+ fromHost = "canary.shop.example.com"
+ fromBody = `{"host":"canary.shop.example.com","from":{"host":"main.shop.example.com"}}`
+)
+
+// fromFixture is a console ready for --from: the app shop (no pool, its own
+// domain), a source instance main with a real litestream replica, a data
+// bucket for seeds, a scratch pool, the store plane armed, and a ready
+// home region.
+type fromFixture struct {
+ fx *consoleFixture
+ fake *objstore.MemProvisioner
+ token string
+ data release.Store
+ url string
+ root string // the replica bucket's root
+ txns int
+}
+
+func fromManifest(private bool) provision.Manifest {
+ return provision.Manifest{Schema: 1, App: fromApp, Domain: fromDomain, Channel: "stable", PrivateStorage: private}
+}
+
+func newFromFixture(t *testing.T, txns int) fromFixture {
+ t.Helper()
+ fx, fake, token, _ := privateStorageFixture(t, "code-from")
+ if _, err := fx.st.CreateAppIn(OpsAccountID, fromApp, "paul@keymail.dev", time.Now()); err != nil {
+ t.Fatal(err)
+ }
+ enableInstances(t, fx, fromApp, fromManifest(false))
+ fx.srv.cfg.HibernateBucket = "data-test"
+ fx.srv.cfg.HibernatePrefix = "instances"
+ fx.srv.cfg.Region = ""
+ data := release.NewDirStore(t.TempDir())
+ fx.srv.cfg.SeedData = data
+ scratch := filepath.Join(t.TempDir(), "scratch", "seeds")
+ if err := clearSeedScratch(scratch); err != nil {
+ t.Fatal(err)
+ }
+ fx.srv.seedScratch = newSeedScratch(scratch, func(string) (int64, error) { return 64 << 30, nil })
+ root := t.TempDir()
+ fx.srv.dbReplica = func(_ context.Context, prefix string) (replicaReader, error) {
+ return lsfile.NewReplicaClient(filepath.Join(root, prefix)), nil
+ }
+ f := fromFixture{fx: fx, fake: fake, token: token, data: data, url: instancesURL(fx, "bab", fromApp), root: root, txns: txns}
+ f.addSource(t, provision.Record{Host: fromSource, Slug: "main"}, txns)
+ publishSeedsFloorFx(t, fx, "v0.27.0")
+ seedReadyBox(t, fx, "kass-1", "", 1, "v0.27.0", "v0.27.0", "nested", false)
+ return f
+}
+
+// addSource writes a source record of shop (rec's other fields as given)
+// and, when txns > 0, a replica of that many batches under its prefix.
+func (f fromFixture) addSource(t *testing.T, rec provision.Record, txns int) {
+ t.Helper()
+ if txns > 0 {
+ dbdoortest.Replicate(t, filepath.Join(f.root, "instances", rec.Host), txns)
+ }
+ rec.Schema, rec.App, rec.Account, rec.Channel, rec.CreatedAt = 1, fromApp, "bab", "stable", time.Now().UTC()
+ if err := provision.CreateRecord(context.Background(), f.fx.rel, rec); err != nil {
+ t.Fatal(err)
+ }
+}
+
+func (f fromFixture) create(t *testing.T, body string) (int, map[string]any) {
+ t.Helper()
+ return f.createAs(t, f.token, body)
+}
+
+func (f fromFixture) createAs(t *testing.T, token, body string) (int, map[string]any) {
+ t.Helper()
+ code, _, raw := rawReq(t, http.MethodPost, f.url, token, body)
+ var out map[string]any
+ _ = json.Unmarshal([]byte(raw), &out)
+ return code, out
+}
+
+func (f fromFixture) record(t *testing.T, host string) (*provision.Record, error) {
+ t.Helper()
+ return provision.GetRecord(context.Background(), f.fx.rel, "bab", fromApp, host)
+}
+
+// requireAborted: the record exists with this door's seed aborted.
+func (f fromFixture) requireAborted(t *testing.T, host string) {
+ t.Helper()
+ rec, err := f.record(t, host)
+ if err != nil || rec.Seed == nil || rec.Seed.State != seeds.StateAborted || rec.Seed.Kind != seeds.KindReplica {
+ t.Fatalf("record %s = %+v, %v; want an aborted replica seed", host, rec, err)
+ }
+}
+
+// requireNoRecord: the refusal came before the record was reserved.
+func (f fromFixture) requireNoRecord(t *testing.T, host string) {
+ t.Helper()
+ if _, err := f.record(t, host); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("record %s: err = %v, want none written", host, err)
+ }
+}
+
+// requireClean: the scratch is empty and the pool holds nothing for shop.
+func (f fromFixture) requireClean(t *testing.T) {
+ t.Helper()
+ if ents, _ := os.ReadDir(f.fx.srv.seedScratch.dir()); len(ents) != 0 {
+ t.Fatalf("scratch left behind: %v", ents)
+ }
+ rel, ref := f.fx.srv.seedScratch.reserve("bab/"+fromApp, 1)
+ if ref != nil {
+ t.Fatalf("the reservation was not released: %+v", ref)
+ }
+ rel()
+}
+
+func snapshotRows(t *testing.T, raw []byte) int {
+ t.Helper()
+ path := filepath.Join(t.TempDir(), "snapshot.db")
+ if err := os.WriteFile(path, raw, 0o600); err != nil {
+ t.Fatal(err)
+ }
+ db, err := sql.Open("sqlite", "file:"+path+"?mode=ro")
+ if err != nil {
+ t.Fatal(err)
+ }
+ defer db.Close()
+ var n int
+ if err := db.QueryRow(`SELECT count(*) FROM t`).Scan(&n); err != nil {
+ t.Fatal(err)
+ }
+ return n
+}
+
+func TestFromCreatesAReadySeededInstance(t *testing.T) {
+ f := newFromFixture(t, 30)
+ code, out := f.create(t, `{"host":"canary.shop.example.com","from":{"host":"main.shop.example.com"},"message":"a copy to try the upgrade on"}`)
+ if code != http.StatusCreated {
+ t.Fatalf("create = %d %v", code, out)
+ }
+ rec, err := f.record(t, fromHost)
+ if err != nil || rec.Seed == nil || rec.Seed.State != seeds.StateReady || rec.Seed.Kind != seeds.KindReplica ||
+ rec.Seed.From != fromSource || rec.Seed.TXID == "" || rec.Seed.Bytes == 0 || len(rec.Seed.SHA256) != 64 ||
+ rec.Seed.By != "alice@keymail.dev" || !seeds.ValidID(rec.Seed.ID) {
+ t.Fatalf("record seed = %+v, %v", rec.Seed, err)
+ }
+ if rec.Environment != "" || rec.Slug != "canary" || rec.Channel != "stable" {
+ t.Fatalf("record = %+v; want the ordinary create's fields", rec)
+ }
+ raw, _, err := f.data.Get(context.Background(), seeds.SnapshotKey(rec.Host, rec.Seed.ID))
+ if err != nil {
+ t.Fatalf("snapshot: %v", err)
+ }
+ sum := sha256.Sum256(raw)
+ if hex.EncodeToString(sum[:]) != rec.Seed.SHA256 || int64(len(raw)) != rec.Seed.Bytes {
+ t.Fatalf("snapshot is %d bytes, sha %x; record says %d, %s", len(raw), sum, rec.Seed.Bytes, rec.Seed.SHA256)
+ }
+ if n := snapshotRows(t, raw); n != f.txns*dbdoortest.Rows {
+ t.Fatalf("snapshot holds %d rows, want %d", n, f.txns*dbdoortest.Rows)
+ }
+ seed, _ := out["seed"].(map[string]any)
+ if seed == nil || seed["state"] != seeds.StateReady || seed["id"] != rec.Seed.ID || out["host"] != fromHost {
+ t.Fatalf("response = %v; want the published record", out)
+ }
+ f.requireClean(t)
+ events, err := f.fx.st.Events(OpsAccountID, fromApp, 50)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var rows []string
+ for _, e := range events {
+ if e.Action == "instance-seed" {
+ rows = append(rows, e.Detail+" | "+e.Address)
+ }
+ if e.Action == "instance-create" {
+ t.Errorf("a seeded create also wrote an instance-create row: %+v", e)
+ }
+ }
+ want := "seeded canary.shop.example.com from main.shop.example.com at txid " + rec.Seed.TXID + " | alice@keymail.dev"
+ if len(rows) != 1 || rows[0] != want {
+ t.Fatalf("instance-seed rows = %q, want [%q]", rows, want)
+ }
+}
+
+func TestFromRefusals(t *testing.T) {
+ f := newFromFixture(t, 30)
+ for _, c := range []struct {
+ body string
+ code int
+ want string
+ }{
+ {`{"host":"c1.shop.example.com","from":{"host":"nope.shop.example.com"}}`, http.StatusConflict, "no instance nope.shop.example.com in shop. A copy can only come from an instance of the same app in the same account."},
+ {`{"host":"c2.shop.example.com","kind":"static","from":{"host":"main.shop.example.com"}}`, http.StatusBadRequest, "a seeded instance is an ordinary instance. Drop --kind, --addr and --role."},
+ {`{"host":"c2.shop.example.com","addrs":["127.0.0.1:9000"],"from":{"host":"main.shop.example.com"}}`, http.StatusBadRequest, "a seeded instance is an ordinary instance"},
+ {`{"host":"c2.shop.example.com","role":"replica","from":{"host":"main.shop.example.com"}}`, http.StatusBadRequest, "a seeded instance is an ordinary instance"},
+ {`{"host":"c2.shop.example.com","replicas":["us"],"from":{"host":"main.shop.example.com"}}`, http.StatusBadRequest, "a seeded instance is an ordinary instance"},
+ {`{"host":"main.shop.example.com","from":{"host":"main.shop.example.com"}}`, http.StatusConflict, "instance main.shop.example.com already exists"},
+ } {
+ code, out := f.create(t, c.body)
+ if code != c.code || !strings.Contains(str(t, out, "error"), c.want) {
+ t.Errorf("%s = %d %v, want %d %q", c.body, code, out, c.code, c.want)
+ }
+ }
+ for _, h := range []string{"c1.shop.example.com", "c2.shop.example.com"} {
+ f.requireNoRecord(t, h)
+ }
+ f.requireClean(t)
+}
+
+// --at: a target inside the history restores exactly there; one at or
+// past the end is the latest copy.
+func TestFromAt(t *testing.T) {
+ f := newFromFixture(t, 30)
+ latest, err := dbdoor.PlanRestore(context.Background(), dbdoor.NewClient(replicaDoer(lsfile.NewReplicaClient(filepath.Join(f.root, "instances", fromSource)))), 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ for _, c := range []struct {
+ host string
+ txid uint64
+ want uint64
+ }{
+ {"c3.shop.example.com", 1, 1},
+ {"c4.shop.example.com", 999999, latest.TXID},
+ } {
+ body, _ := json.Marshal(map[string]any{"host": c.host, "from": map[string]any{"host": fromSource, "txid": c.txid}})
+ if code, out := f.create(t, string(body)); code != http.StatusCreated {
+ t.Fatalf("--at %d = %d %v", c.txid, code, out)
+ }
+ rec, err := f.record(t, c.host)
+ if err != nil || rec.Seed == nil || rec.Seed.State != seeds.StateReady || rec.Seed.TXID != strconv.FormatUint(c.want, 10) {
+ t.Fatalf("--at %d: record seed = %+v, %v; want ready at txid %d", c.txid, rec.Seed, err, c.want)
+ }
+ }
+}
+
+func TestFromRefusedBeforeTheFloor(t *testing.T) {
+ f := newFromFixture(t, 10)
+ seedReadyBox(t, f.fx, "kass-1", "", 0, "v0.27.0", "v0.27.0", "nested", false)
+ code, out := f.create(t, fromBody)
+ if code != http.StatusConflict || !strings.Contains(str(t, out, "error"), "below the seeds floor") {
+ t.Fatalf("= %d %v", code, out)
+ }
+ f.requireNoRecord(t, fromHost)
+}
+
+// --from is refused while host claims are armed: replicaDoer has no
+// pinned-version branch, and the seed floor's own claim rule refuses
+// first, before any record is written (Task 4's carried item).
+func TestFromRefusedWithClaimsArmed(t *testing.T) {
+ f := newFromFixture(t, 10)
+ f.fx.srv.cfg.HostClaim = true
+ code, out := f.create(t, fromBody)
+ if code != http.StatusConflict || str(t, out, "error") != "This console runs with claim control on. Seeded instances are not supported here yet." {
+ t.Fatalf("= %d %v", code, out)
+ }
+ f.requireNoRecord(t, fromHost)
+}
+
+func TestFromTooLarge(t *testing.T) {
+ f := newFromFixture(t, 10)
+ old := seedMaxBytes
+ seedMaxBytes = 1024 // the test's database is larger than 1 KiB
+ defer func() { seedMaxBytes = old }()
+ code, out := f.create(t, fromBody)
+ if code != http.StatusRequestEntityTooLarge || !strings.Contains(str(t, out, "error"), "a copy of main.shop.example.com would be ") ||
+ !strings.Contains(str(t, out, "error"), " GiB, over the 4 GiB limit.") {
+ t.Fatalf("= %d %v", code, out)
+ }
+ f.requireNoRecord(t, fromHost)
+ f.requireClean(t)
+}
+
+func TestFromSourceRefusals(t *testing.T) {
+ f := newFromFixture(t, 3)
+ f.addSource(t, provision.Record{Host: "pooled.shop.example.com", Slug: "pooled", Pool: "shared"}, 0)
+ f.addSource(t, provision.Record{Host: "fleet.shop.example.com", Slug: "fleet", Region: "fleet/racks"}, 0)
+ f.addSource(t, provision.Record{Host: "site.shop.example.com", Slug: "site", Kind: "static"}, 0)
+ f.addSource(t, provision.Record{Host: "far.shop.example.com", Slug: "far", Region: "us"}, 0)
+ f.addSource(t, provision.Record{Host: "empty.shop.example.com", Slug: "empty"}, 0)
+ for _, c := range []struct {
+ source string
+ code int
+ want string
+ }{
+ {"pooled.shop.example.com", http.StatusConflict, "pooled.shop.example.com runs in the pool shared, so the platform keeps no replica of it to copy."},
+ {"fleet.shop.example.com", http.StatusConflict, "fleet.shop.example.com runs on your fleet racks, so the platform keeps no replica of it to copy."},
+ {"site.shop.example.com", http.StatusConflict, "site.shop.example.com is not an instance the platform keeps a replica of, so it cannot be copied."},
+ {"far.shop.example.com", http.StatusConflict, "far.shop.example.com keeps its data in "},
+ {"empty.shop.example.com", http.StatusConflict, "empty.shop.example.com has not replicated anything yet. Wake it once, then try again."},
+ } {
+ code, out := f.create(t, `{"host":"canary.shop.example.com","from":{"host":"`+c.source+`"}}`)
+ if code != c.code || !strings.Contains(str(t, out, "error"), c.want) {
+ t.Errorf("from %s = %d %v, want %d %q", c.source, code, out, c.code, c.want)
+ }
+ }
+ f.requireNoRecord(t, fromHost)
+ f.requireClean(t)
+}
+
+// A copy stays in its source's region: a target homed elsewhere is
+// refused, before any record is written.
+func TestFromRefusesAnotherRegion(t *testing.T) {
+ f := newFromFixture(t, 3)
+ seedReadyBox(t, f.fx, "far-1", "us", 1, "v0.27.0", "v0.27.0", "nested", false)
+ code, out := f.create(t, `{"host":"canary.shop.example.com","region":"us","from":{"host":"main.shop.example.com"}}`)
+ if code != http.StatusConflict || str(t, out, "error") != "a copy stays in its source's region. To run it elsewhere, create it here and then run carlos move." {
+ t.Fatalf("= %d %v", code, out)
+ }
+ f.requireNoRecord(t, fromHost)
+}
+
+func TestFromPlanFailures(t *testing.T) {
+ f := newFromFixture(t, 3)
+ oldPlan := planRestore
+ defer func() { planRestore = oldPlan }()
+ for i, c := range []struct {
+ err error
+ body string
+ code int
+ want string
+ }{
+ {dbdoor.ErrHistoryGone, `{"host":"canary.shop.example.com","from":{"host":"main.shop.example.com","txid":2}}`, http.StatusConflict, "history before 2 is no longer kept"},
+ {dbdoor.ErrNothingReplicated, `{"host":"canary.shop.example.com","from":{"host":"main.shop.example.com","txid":2}}`, http.StatusConflict, "history before 2 is no longer kept"},
+ {dbdoor.ErrNothingReplicated, fromBody, http.StatusConflict, "main.shop.example.com has not replicated anything yet. Wake it once, then try again."},
+ {errors.New("dbdoor: the last planned file's header does not describe it"), fromBody, http.StatusBadGateway, "could not copy main.shop.example.com. Nothing was created. Try again."},
+ {os.ErrNotExist, fromBody, http.StatusBadGateway, "could not copy main.shop.example.com. Nothing was created. Try again."},
+ } {
+ planRestore = func(context.Context, *dbdoor.Client, uint64) (dbdoor.RestorePlan, error) {
+ return dbdoor.RestorePlan{}, c.err
+ }
+ code, out := f.create(t, c.body)
+ if code != c.code || str(t, out, "error") != c.want {
+ t.Errorf("%d: %v = %d %v, want %d %q", i, c.err, code, out, c.code, c.want)
+ }
+ }
+ f.requireNoRecord(t, fromHost)
+ f.requireClean(t)
+}
+
+// Every failure after the record is reserved aborts it, answers its own
+// sentence, and leaves nothing in scratch or the pool.
+func TestFromRestoreFailuresAbort(t *testing.T) {
+ oldRestore, oldDeadline := restoreToTarget, seedRestoreDeadline
+ defer func() { restoreToTarget, seedRestoreDeadline = oldRestore, oldDeadline }()
+ for i, c := range []struct {
+ restore func(ctx context.Context) error
+ body string
+ code int
+ want string
+ }{
+ {func(context.Context) error { return dbdoor.ErrCheckFailed }, fromBody, http.StatusUnprocessableEntity, "the copy of main.shop.example.com failed its integrity check, so no instance was created."},
+ {func(context.Context) error { return dbdoor.ErrHistoryGone }, fromBody, http.StatusBadGateway, "could not copy main.shop.example.com. Nothing was created. Try again."},
+ {func(context.Context) error { return dbdoor.ErrHistoryGone }, `{"host":"canary.shop.example.com","from":{"host":"main.shop.example.com","txid":2}}`, http.StatusConflict, "history before 2 is no longer kept"},
+ {func(context.Context) error { return errors.New("bucket read failed") }, fromBody, http.StatusBadGateway, "could not copy main.shop.example.com. Nothing was created. Try again."},
+ {func(ctx context.Context) error { <-ctx.Done(); return ctx.Err() }, fromBody, http.StatusGatewayTimeout, "copying main.shop.example.com took longer than 30 minutes, so no instance was created."},
+ } {
+ f := newFromFixture(t, 3)
+ seedRestoreDeadline = 200 * time.Millisecond
+ restoreToTarget = func(ctx context.Context, _ *dbdoor.Client, parent string, _ uint64) (*dbdoor.Restored, error) {
+ return nil, c.restore(ctx)
+ }
+ code, out := f.create(t, c.body)
+ if code != c.code || str(t, out, "error") != c.want {
+ t.Errorf("%d: = %d %v, want %d %q", i, code, out, c.code, c.want)
+ continue
+ }
+ f.requireAborted(t, fromHost)
+ f.requireClean(t)
+ }
+}
+
+// The restore is pinned at the planned TXID, whatever the request said:
+// only that restore produces the size that was admitted.
+func TestFromRestoresAtThePlannedTXID(t *testing.T) {
+ f := newFromFixture(t, 5)
+ oldRestore := restoreToTarget
+ defer func() { restoreToTarget = oldRestore }()
+ var got []uint64
+ restoreToTarget = func(ctx context.Context, c *dbdoor.Client, parent string, target uint64) (*dbdoor.Restored, error) {
+ got = append(got, target)
+ return oldRestore(ctx, c, parent, target)
+ }
+ plan, err := dbdoor.PlanRestore(context.Background(), dbdoor.NewClient(replicaDoer(lsfile.NewReplicaClient(filepath.Join(f.root, "instances", fromSource)))), 0)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if code, out := f.create(t, fromBody); code != http.StatusCreated {
+ t.Fatalf("= %d %v", code, out)
+ }
+ if code, out := f.create(t, `{"host":"c2.shop.example.com","from":{"host":"main.shop.example.com","txid":999999}}`); code != http.StatusCreated {
+ t.Fatalf("= %d %v", code, out)
+ }
+ if len(got) != 2 || got[0] != plan.TXID || got[1] != plan.TXID {
+ t.Fatalf("restored at %v, want [%d %d]", got, plan.TXID, plan.TXID)
+ }
+}
+
+// A restore that finished larger than planned is refused after the fact
+// (the watcher's tick can come too late for a fast restore).
+func TestFromRefusesARestoreLargerThanPlanned(t *testing.T) {
+ f := newFromFixture(t, 30)
+ oldPlan := planRestore
+ planRestore = func(ctx context.Context, c *dbdoor.Client, target uint64) (dbdoor.RestorePlan, error) {
+ p, err := oldPlan(ctx, c, target)
+ p.RestoredBytes = 1 // admit far less than the restore will write
+ return p, err
+ }
+ defer func() { planRestore = oldPlan }()
+ code, out := f.create(t, fromBody)
+ if code != http.StatusRequestEntityTooLarge || str(t, out, "error") != "the copy of main.shop.example.com grew past its planned size, so it was stopped and no instance was created." {
+ t.Fatalf("= %d %v", code, out)
+ }
+ f.requireAborted(t, fromHost)
+ f.requireClean(t)
+ if keys, err := f.data.List(context.Background(), seeds.Prefix); err != nil || len(keys) != 0 {
+ t.Fatalf("seeds/ = %v, %v; want nothing uploaded", keys, err)
+ }
+}
+
+// The watcher cancels a restore that is still running: litestream writes
+// replica.db.tmp and renames it only at the end, so the watcher must see
+// the .tmp file grow.
+func TestFromWatcherCancelsARestoreInFlight(t *testing.T) {
+ f := newFromFixture(t, 3)
+ oldRestore, oldEvery := restoreToTarget, restoreWatchEvery
+ defer func() { restoreToTarget, restoreWatchEvery = oldRestore, oldEvery }()
+ restoreWatchEvery = 20 * time.Millisecond
+ cancelled := make(chan bool, 1)
+ restoreToTarget = func(rctx context.Context, _ *dbdoor.Client, parent string, _ uint64) (*dbdoor.Restored, error) {
+ dir, err := os.MkdirTemp(parent, "carlos-db-")
+ if err != nil {
+ return nil, err
+ }
+ defer os.RemoveAll(dir)
+ out, err := os.Create(filepath.Join(dir, "replica.db.tmp"))
+ if err != nil {
+ return nil, err
+ }
+ defer out.Close()
+ chunk := make([]byte, 64<<10)
+ giveUp := time.After(10 * time.Second)
+ for written := 0; ; {
+ if written < 4<<20 { // well past the database's planned size, and bounded
+ n, _ := out.Write(chunk)
+ written += n
+ }
+ select {
+ case <-rctx.Done():
+ cancelled <- true
+ return nil, rctx.Err()
+ case <-giveUp:
+ cancelled <- false
+ return nil, errors.New("the watcher never cancelled the restore")
+ case <-time.After(10 * time.Millisecond):
+ }
+ }
+ }
+ code, out := f.create(t, fromBody)
+ if !<-cancelled {
+ t.Fatal("the restore ran to its own timeout: the watcher did not cancel it in flight")
+ }
+ if code != http.StatusRequestEntityTooLarge || str(t, out, "error") != "the copy of main.shop.example.com grew past its planned size, so it was stopped and no instance was created." {
+ t.Fatalf("= %d %v", code, out)
+ }
+ f.requireAborted(t, fromHost)
+ f.requireClean(t)
+}
+
+func TestFromDisconnectAbortsTheRecordAndCleansScratch(t *testing.T) {
+ f := newFromFixture(t, 30)
+ ctx, cancel := context.WithCancel(context.Background())
+ oldRestore := restoreToTarget
+ restoreToTarget = func(rctx context.Context, c *dbdoor.Client, parent string, target uint64) (*dbdoor.Restored, error) {
+ if _, err := os.MkdirTemp(parent, "carlos-db-"); err != nil { // scratch the door must clear
+ return nil, err
+ }
+ cancel() // the client goes away mid-restore
+ <-rctx.Done()
+ return nil, rctx.Err()
+ }
+ defer func() { restoreToTarget = oldRestore }()
+ req, _ := http.NewRequestWithContext(ctx, http.MethodPost, f.url, strings.NewReader(fromBody))
+ req.Header.Set("Authorization", "Bearer "+f.token)
+ req.Header.Set("Content-Type", "application/json")
+ if res, err := http.DefaultClient.Do(req); err == nil {
+ res.Body.Close()
+ t.Fatalf("the request finished with %d; want the client to have gone away", res.StatusCode)
+ }
+ deadline := time.Now().Add(5 * time.Second)
+ for time.Now().Before(deadline) {
+ rec, err := f.record(t, fromHost)
+ if err == nil && rec.Seed != nil && rec.Seed.State == seeds.StateAborted {
+ if ents, _ := os.ReadDir(f.fx.srv.seedScratch.dir()); len(ents) == 0 {
+ if rel, ref := f.fx.srv.seedScratch.reserve("bab/"+fromApp, 1); ref == nil {
+ rel()
+ return
+ }
+ }
+ }
+ time.Sleep(20 * time.Millisecond)
+ }
+ t.Fatal("a disconnected --from did not abort its record, clear its scratch and release its reservation")
+}
+
+// A copy that runs longer than the server's ReadTimeout still completes.
+// (On Go 1.26 net/http clears the read deadline once the body is read, so
+// this holds with or without the door's own extension; it pins the
+// behaviour, not the mechanism.)
+func TestFromOutlivesTheServerReadTimeout(t *testing.T) {
+ f := newFromFixture(t, 3)
+ ts := httptest.NewUnstartedServer(f.fx.srv)
+ ts.Config.ReadTimeout = 300 * time.Millisecond
+ ts.Start()
+ defer ts.Close()
+ oldRestore := restoreToTarget
+ defer func() { restoreToTarget = oldRestore }()
+ restoreToTarget = func(ctx context.Context, c *dbdoor.Client, parent string, target uint64) (*dbdoor.Restored, error) {
+ select {
+ case <-ctx.Done():
+ return nil, ctx.Err()
+ case <-time.After(1200 * time.Millisecond): // four times the server's ReadTimeout
+ }
+ return oldRestore(ctx, c, parent, target)
+ }
+ code, _, raw := rawReq(t, http.MethodPost, ts.URL+"/api/cli/apps/bab/shop/instances", f.token, fromBody)
+ if code != http.StatusCreated {
+ t.Fatalf("a copy longer than the ReadTimeout = %d %s, want 201", code, raw)
+ }
+}
+
+func TestFromUploadFailureAborts(t *testing.T) {
+ f := newFromFixture(t, 3)
+ f.fx.srv.cfg.SeedData = failingPuts{f.data}
+ code, out := f.create(t, fromBody)
+ if code != http.StatusBadGateway || str(t, out, "error") != "could not copy main.shop.example.com. Nothing was created. Try again." {
+ t.Fatalf("= %d %v", code, out)
+ }
+ f.requireAborted(t, fromHost)
+ f.requireClean(t)
+}
+
+type failingPuts struct{ release.Store }
+
+func (failingPuts) Put(context.Context, string, []byte, string) (string, error) {
+ return "", errors.New("access denied")
+}
+
+func TestFromNeedsADataBucket(t *testing.T) {
+ f := newFromFixture(t, 3)
+ f.fx.srv.cfg.SeedData = nil
+ code, out := f.create(t, fromBody)
+ if code != http.StatusServiceUnavailable || str(t, out, "error") != "this console has no data bucket set up, so it cannot create a seeded instance." {
+ t.Fatalf("= %d %v", code, out)
+ }
+ f.requireNoRecord(t, fromHost)
+}
+
+func TestFromRefusesAPooledApp(t *testing.T) {
+ f := newFromFixture(t, 3)
+ m := fromManifest(false)
+ m.Pool = "shared"
+ enableInstances(t, f.fx, fromApp, m)
+ code, out := f.create(t, fromBody)
+ if code != http.StatusConflict || str(t, out, "error") != "shop places its instances in a pool or on a fleet, so it cannot take a seeded instance." {
+ t.Fatalf("= %d %v", code, out)
+ }
+ f.requireNoRecord(t, fromHost)
+}
+
+// The instance service role reaches --from through the create door's own
+// route, as spec §6.1 says; the db door stays person-only.
+func TestFromByAnInstanceServiceRole(t *testing.T) {
+ f := newFromFixture(t, 3)
+ svc := serviceToken(t, f.fx.st, OpsAccountID, "ci", fromApp, svcrole.Instance)
+ if code, out := f.createAs(t, svc, fromBody); code != http.StatusCreated {
+ t.Fatalf("= %d %v", code, out)
+ }
+ rec, err := f.record(t, fromHost)
+ if err != nil || rec.Seed == nil || rec.Seed.State != seeds.StateReady || rec.Seed.By != "service:bab/ci" {
+ t.Fatalf("record seed = %+v, %v", rec.Seed, err)
+ }
+}
+
+func TestFromPrivateStorageRules(t *testing.T) {
+ f := newFromFixture(t, 3)
+ enableInstances(t, f.fx, fromApp, fromManifest(true))
+ code, out := f.create(t, `{"host":"canary.shop.example.com","environment":"main","from":{"host":"main.shop.example.com"}}`)
+ if code != http.StatusConflict || str(t, out, "error") != "a seeded instance gets its own environment and an empty store" {
+ t.Fatalf("explicit environment = %d %v", code, out)
+ }
+ // A store already provisioned for the derived environment, by no seed.
+ at := time.Now().UTC().Add(-time.Hour)
+ putCanaryStoreOf(t, f.fx, fromApp, &objstore.Record{
+ Declared: &objstore.Declared{EnvPrefix: "S3", At: at, By: "bob@keymail.dev"},
+ Grant: &objstore.GrantBlock{Backend: "mem", Bucket: "carlos-bab-shop-canary", Source: objstore.GrantSourceAccountApproval, At: at, By: "operator@keymail.dev"},
+ })
+ code, out = f.create(t, fromBody)
+ if code != http.StatusConflict || str(t, out, "error") != "environment canary already has a store (carlos-bab-shop-canary). A seeded instance needs a new, empty one." {
+ t.Fatalf("an existing store = %d %v", code, out)
+ }
+ f.requireNoRecord(t, fromHost)
+ f.requireClean(t)
+}
+
+func putCanaryStoreOf(t *testing.T, fx *consoleFixture, app string, rec *objstore.Record) {
+ t.Helper()
+ if _, err := objstore.PutIn(context.Background(), fx.rel, release.OpsAccount, app, "canary", rec, ""); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// With private storage the copy gets a new store marked with its seed, and
+// the source's own store is not read, bound or touched.
+func TestFromPrivateStorageGetsANewStoreMarkedWithTheSeed(t *testing.T) {
+ f := newFromFixture(t, 3)
+ enableInstances(t, f.fx, fromApp, fromManifest(true))
+ at := time.Now().UTC().Add(-time.Hour)
+ if _, err := objstore.PutIn(context.Background(), f.fx.rel, release.OpsAccount, fromApp, "main", &objstore.Record{
+ Declared: &objstore.Declared{EnvPrefix: "S3", At: at, By: "bob@keymail.dev"},
+ Grant: &objstore.GrantBlock{Backend: "mem", Bucket: "carlos-bab-shop-main", Source: objstore.GrantSourceAccountApproval, At: at, By: "operator@keymail.dev"},
+ }, ""); err != nil {
+ t.Fatal(err)
+ }
+ mainKey := objstore.KeyIn(release.OpsAccount, fromApp, "main")
+ before, _, err := f.fx.rel.Get(context.Background(), mainKey)
+ if err != nil {
+ t.Fatal(err)
+ }
+ code, out := f.create(t, fromBody)
+ if code != http.StatusCreated {
+ t.Fatalf("= %d %v", code, out)
+ }
+ rec, err := f.record(t, fromHost)
+ if err != nil || rec.Seed == nil || rec.Seed.State != seeds.StateReady || rec.Environment != "canary" {
+ t.Fatalf("record = %+v, %v", rec, err)
+ }
+ store, _, err := objstore.GetIn(context.Background(), f.fx.rel, release.OpsAccount, fromApp, "canary")
+ if err != nil || store == nil || store.Declared == nil || store.Declared.Seed != rec.Seed.ID || storeBucketName(store) == "" {
+ t.Fatalf("canary store = %+v, %v; want one marked with seed %s", store, err, rec.Seed.ID)
+ }
+ if out["private_store_bucket"] != storeBucketName(store) {
+ t.Fatalf("response private_store_bucket = %v, want %s", out["private_store_bucket"], storeBucketName(store))
+ }
+ after, _, err := f.fx.rel.Get(context.Background(), mainKey)
+ if err != nil || string(after) != string(before) {
+ t.Fatalf("the source's store record changed:\n before %s\n after %s (%v)", before, after, err)
+ }
+}
+
+// The create door's sibling-apex re-check holds on the long --from path
+// too: an app created under the host's label while the copy ran owns its
+// apex, and the seed is aborted rather than published.
+func TestFromAbortsWhenASiblingTakesTheApex(t *testing.T) {
+ f := newFromFixture(t, 3)
+ f.fx.srv.cfg.AppsDomain = "apps.example.net"
+ m := fromManifest(false)
+ m.Domain = "bab.apps.example.net"
+ enableInstances(t, f.fx, fromApp, m)
+ f.addSource(t, provision.Record{Host: "main.bab.apps.example.net", Slug: "main"}, 3)
+ oldRestore := restoreToTarget
+ defer func() { restoreToTarget = oldRestore }()
+ restoreToTarget = func(ctx context.Context, c *dbdoor.Client, parent string, target uint64) (*dbdoor.Restored, error) {
+ if _, err := f.fx.st.CreateAppIn(OpsAccountID, "canary", "paul@keymail.dev", time.Now()); err != nil {
+ return nil, err
+ }
+ return oldRestore(ctx, c, parent, target)
+ }
+ code, out := f.create(t, `{"host":"canary.bab.apps.example.net","from":{"host":"main.bab.apps.example.net"}}`)
+ want := "canary.bab.apps.example.net is app canary's platform apex — it was created while this instance was being written; the instance was not kept"
+ if code != http.StatusConflict || str(t, out, "error") != want {
+ t.Fatalf("= %d %v", code, out)
+ }
+ f.requireAborted(t, "canary.bab.apps.example.net")
+ f.requireClean(t)
+}
+
+func TestRestoreScratchUse(t *testing.T) {
+ dir := t.TempDir()
+ sub := filepath.Join(dir, "carlos-db-x")
+ if err := os.Mkdir(sub, 0o700); err != nil {
+ t.Fatal(err)
+ }
+ for name, n := range map[string]int{"replica.db.tmp": 100, "ltx-1": 40, "ltx-2": 60, "replica.db-shm": 32768} {
+ if err := os.WriteFile(filepath.Join(sub, name), make([]byte, n), 0o600); err != nil {
+ t.Fatal(err)
+ }
+ }
+ db, total := restoreScratchUse(dir)
+ if db != 100 || total != 200 {
+ t.Fatalf("restoreScratchUse = %d, %d; want 100, 200 (SQLite's own -shm is not the restore's)", db, total)
+ }
+}
diff --git a/internal/console/claim_authority_census_test.go b/internal/console/claim_authority_census_test.go
index 88b13b8dc..10710d44c 100644
--- a/internal/console/claim_authority_census_test.go
+++ b/internal/console/claim_authority_census_test.go
@@ -21,7 +21,7 @@ func claimAuthoritySites(t *testing.T) []string {
"NewDirStore": true, "Open": true, "OpenDB": true,
"NewConditionalRecordStore": true, "NewVersionedBoundary": true, "NewFromConfig": true, "S3ClientFor": true, "NewControlWriter": true, "NewRestoreControl": true, "NewMutationBoundary": true,
"NewObservedMutationBoundary": true, "NewProductionMutationCheck": true,
- "NewS3Store": true, "NewVersionedS3": true, "NewAWSProvisioner": true,
+ "NewS3Store": true, "NewS3StoreIn": true, "NewVersionedS3": true, "NewAWSProvisioner": true,
"NewClamAVScanner": true, "NewRoute53Writer": true, "BeginExclusive": true,
"ResumeExclusive": true, "RegisterStart": true, "FinishWakeRetryReset": true, "ReservePurge": true,
"BeginPurge": true, "SavePurgeJournal": true, "RemovePurgeJournal": true,
diff --git a/internal/console/seedfloor.go b/internal/console/seedfloor.go
index 2cb09b527..a04e78a23 100644
--- a/internal/console/seedfloor.go
+++ b/internal/console/seedfloor.go
@@ -35,7 +35,7 @@ import (
func (s *Server) seedRegionReady(ctx context.Context, region string, oneBox bool) *doorRefusal {
region = s.homeRegionIn(region)
if s.cfg.HostClaim {
- return &doorRefusal{http.StatusConflict, "This console runs with claim control on. Seeded instances are not supported here yet."}
+ return seedClaimConsoleRefusal()
}
floor, err := seeds.ReadFloor(ctx, s.cfg.Releases)
if err == nil && floor != nil {
@@ -148,6 +148,13 @@ func (s *Server) seedRegionReady(ctx context.Context, region string, oneBox bool
return nil
}
+// seedClaimConsoleRefusal is the seed doors' answer on a console with claim
+// control on. The --from door also gives it for a source whose claim could
+// not be read, which seedRegionReady's own check makes unreachable today.
+func seedClaimConsoleRefusal() *doorRefusal {
+ return &doorRefusal{http.StatusConflict, "This console runs with claim control on. Seeded instances are not supported here yet."}
+}
+
// boxList is the refusals' box names: sorted, comma-joined.
func boxList(boxes []string) string {
sort.Strings(boxes)
diff --git a/internal/console/seedscratch.go b/internal/console/seedscratch.go
index a1d4c33a4..011435054 100644
--- a/internal/console/seedscratch.go
+++ b/internal/console/seedscratch.go
@@ -89,7 +89,7 @@ func (p *seedScratch) reserve(appKey string, bytes int64) (func(), *doorRefusal)
p.readCapacity()
}
if bytes < 0 || bytes > p.capacity-p.used {
- return nil, &doorRefusal{http.StatusInsufficientStorage, "the console has no room for this right now. Try again later."}
+ return nil, seedScratchFull()
}
p.used += bytes
p.running++
@@ -106,6 +106,12 @@ func (p *seedScratch) reserve(appKey string, bytes int64) (func(), *doorRefusal)
}, nil
}
+// seedScratchFull is the 507 the pool answers, and what a door answers
+// when it cannot make its directory in the scratch.
+func seedScratchFull() *doorRefusal {
+ return &doorRefusal{http.StatusInsufficientStorage, "the console has no room for this right now. Try again later."}
+}
+
// clearSeedScratch removes what a crashed restore or spool left at console
// start (spec §4 "Cleanup", §5.2): the entries named seedScratchPrefix* that
// sit directly inside dir. Anything else in dir stays, and dir itself is
diff --git a/internal/console/server.go b/internal/console/server.go
index 828b00994..f728d69c2 100644
--- a/internal/console/server.go
+++ b/internal/console/server.go
@@ -483,9 +483,13 @@ type Config struct {
// import spools. Start-up removes the pool's own leftovers from it.
// Empty: the seed doors answer 507.
SeedScratchDir string
- DBVersions lease.VersionedStore
- Origin string // the console's public origin — the OAuth client_id
- Keymail string // keymail base URL
+ // SeedData is the home region's data bucket, where seeds/ lives (the
+ // bucket HibernateBucket names, at its root). nil: the seed doors
+ // answer 503.
+ SeedData release.Store
+ DBVersions lease.VersionedStore
+ Origin string // the console's public origin — the OAuth client_id
+ Keymail string // keymail base URL
// Registry is the path to the platform registry DB. Read-mostly (routes,
// domains, alerts), but fleets.go's fleet/box/placement doors (customer
// fleets plan, Task 7) genuinely write through it via openRegistry
diff --git a/internal/console/testdata/claim-authority-sites.txt b/internal/console/testdata/claim-authority-sites.txt
index 071c68898..ea5725bc6 100644
--- a/internal/console/testdata/claim-authority-sites.txt
+++ b/internal/console/testdata/claim-authority-sites.txt
@@ -2,6 +2,7 @@
../../cmd/console/main.go|openStore|release.NewDirStore(d)
../../cmd/console/main.go|openStore|release.NewS3Store(context.Background(), b)
../../cmd/console/main.go|serve|lease.NewVersionedS3(context.Background(), buckets)
+../../cmd/console/main.go|serve|release.NewS3StoreIn(context.Background(), dbBucket, dbRegion)
../../cmd/console/main.go|storeBackend|objstore.NewAWSProvisioner(context.Background(), region, endpoint, boundary)
../../cmd/console/main.go|storeScannerFor|objstore.NewClamAVScanner(context.Background(), d.Region, d.Endpoint, addr, maxBytes, signer)
../hostops/app_mutation_runner.go|RunAppMutation|b.control.BeginExclusive(ctx, account, app, a.Value.AppID, AppOperation{ID: id, Kind: OpAppMutation, AppID: a.Value.AppID, Phase: "issued", Intent: intent}, owner)
@@ -26,6 +27,7 @@
../mail/aws.go|NewAWSProvisioner|iam.NewFromConfig(cfg)
../mail/aws.go|NewAWSProvisioner|sesv2.NewFromConfig(cfg)
../mail/aws.go|NewAWSProvisioner|sts.NewFromConfig(cfg)
+apiinstanceseed.go|uploadSnapshot|os.Open(path)
apiruns.go|handleAPIRunCreate|rec.ActiveRun.Open()
apiruns.go|writeRunObjects|os.Open(f.path)
apirunsfollow.go|handleAPIRunAck|run.Open()