amadan

carlos / platform Public

console: finishSeed, the seed-guarded store teardown and the record delete last (Plan 6 Task 4)

finishSeed frees a host once its seed is rolled-back: under the store's
walk lock it re-reads the environment's store and tears it down only
while the store's seed mark is this seed and no other instance names the
environment (preflight M9, M10); it deletes the seed's snapshot and, for
a --from seed, every key under the host's replica prefix (M11); and it
deletes the record last, by CAS on the etag of the read that saw the
seed rolled-back. seeds.Check runs before any key is built (M8). Every
step is idempotent, so a finish that fails part way is resumed by the
next one. Tests run the real TeardownIn against the MemProvisioner (M12)
with neutral names (M13).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
claude 677796be99ac08b84a4646e17170cb0e4ce5abad parent 248aa34
2 files changed, +615 −0
  • internal/console/seedfinish.go +175 −0
  • internal/console/seedfinish_test.go +440 −0
diff --git a/internal/console/seedfinish.go b/internal/console/seedfinish.go
new file mode 100644
index 000000000..22b173641
--- /dev/null
+++ b/internal/console/seedfinish.go
@@ -0,0 +1,175 @@
+// The console's finish for an abandoned seed (seeded-instances spec §5.6,
+// codex R6-1). Once a box has rolled a seed back, or the delete door has
+// moved an import no box swapped straight to rolled-back, the record still
+// holds the hostname. finishSeed is what frees it: the store the seed made,
+// the seed's snapshot, a --from seed's replica, and the record last.
+
+package console
+
+import (
+ "context"
+ "errors"
+ "fmt"
+ "log"
+ "os"
+
+ "github.com/carlosframework/platform/internal/activator"
+ "github.com/carlosframework/platform/internal/objstore"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/seeds"
+)
+
+// finishSeed finishes seedID on host's record, only while that record
+// still names seedID and says rolled-back. Any other record (another seed,
+// another state, none) is left alone and answers nil. In order:
+//
+// 1. The store of the record's environment, under the store's walk lock
+// and re-read there, is torn down only while its Declared.Seed is
+// seedID (preflight M9), and only when no other instance names that
+// environment (M10). A store with another seed, or none, is never
+// touched.
+// 2. The seed's snapshot, seeds/<host>/<id>/snapshot.db.
+// 3. For a --from seed, every key under the host's replica prefix in the
+// data bucket (M11). It is this seed's: the record is create-only and
+// the create door's leftover-replica rail refused the host if anything
+// was there, so only this seed's instance has written there since.
+// 4. The record, by CAS on the etag of the read that saw this seed in
+// rolled-back. It goes last because until then it holds the hostname,
+// so a re-import of the host never meets a leftover store.
+//
+// Every step is idempotent: a finish that fails part way answers an error
+// with the record still rolled-back, and the next finish (the caller's
+// retry, or the sweep) resumes it. seeds.Check runs before any key is
+// built (M8). ctx bounds the whole finish; the teardown also has its own
+// storeTeardownTimeout. Callers that finish inline on a request's behalf
+// pass context.WithoutCancel of it, so a departing client cannot cut a
+// teardown midway.
+func (s *Server) finishSeed(ctx context.Context, sqid, app, host, seedID string) error {
+ if err := seeds.Check(host, seedID); err != nil {
+ return fmt.Errorf("finish seed %q of %q: %w", seedID, host, err)
+ }
+ err := s.finishSeedOnce(ctx, sqid, app, host, seedID)
+ if errors.Is(err, release.ErrCASConflict) {
+ // Another writer changed the record between the read and the
+ // delete. Every step is idempotent, so read it again once.
+ err = s.finishSeedOnce(ctx, sqid, app, host, seedID)
+ }
+ return err
+}
+
+func (s *Server) finishSeedOnce(ctx context.Context, sqid, app, host, seedID string) error {
+ rec, etag, err := provision.GetRecordETag(ctx, s.cfg.Releases, sqid, app, host)
+ if errors.Is(err, os.ErrNotExist) {
+ return nil
+ }
+ if err != nil {
+ return fmt.Errorf("read the record of %s: %w", host, err)
+ }
+ if rec.Seed == nil || rec.Seed.ID != seedID || rec.Seed.State != seeds.StateRolledBack {
+ return nil
+ }
+ if err := s.finishSeedStore(ctx, sqid, app, host, seedID, rec.Environment); err != nil {
+ return err
+ }
+ if s.cfg.SeedData == nil {
+ return fmt.Errorf("finish seed %s of %s: this console has no data bucket to delete its snapshot from", seedID, host)
+ }
+ if err := s.cfg.SeedData.Delete(ctx, seeds.SnapshotKey(host, seedID)); err != nil && !errors.Is(err, os.ErrNotExist) {
+ return fmt.Errorf("delete the snapshot of seed %s of %s: %w", seedID, host, err)
+ }
+ if rec.Seed.Kind == seeds.KindReplica {
+ if err := s.deleteSeedReplica(ctx, rec); err != nil {
+ return err
+ }
+ }
+ if err := release.DeleteIfMatch(ctx, s.cfg.Releases, provision.RecordKey(sqid, app, host), etag); err != nil && !errors.Is(err, os.ErrNotExist) {
+ return fmt.Errorf("delete the record of %s: %w", host, err)
+ }
+ log.Printf("console: finished seed %s of %s/%s/%s", seedID, sqid, app, host)
+ s.bumpSerial(ctx)
+ return nil
+}
+
+// finishSeedStore is the finish's first step: the store of env, torn down
+// only while seedID made it and no other instance uses it. It holds the
+// store's walk lock throughout, so a grant (a begin's, or carlos store
+// create --environment) cannot change the store between the check and the
+// teardown.
+func (s *Server) finishSeedStore(ctx context.Context, sqid, app, host, seedID, env string) error {
+ defer s.storeWalks.lock(sqid, app, env)()
+ store, _, err := objstore.GetIn(ctx, s.cfg.Releases, sqid, app, env)
+ if err != nil {
+ return fmt.Errorf("read the store of %s: %w", host, err)
+ }
+ if store == nil || store.Declared == nil || store.Declared.Seed != seedID {
+ return nil
+ }
+ other, err := s.otherInstanceInEnvironment(ctx, sqid, app, host, env)
+ if err != nil {
+ return err
+ }
+ if other != "" {
+ log.Printf("console: finish seed %s of %s/%s/%s: OPERATOR: instance %s also uses environment %q, so the store seed %s made (bucket %s) is left standing; tear it down by hand if it is not wanted",
+ seedID, sqid, app, host, other, env, seedID, storeBucketName(store))
+ return nil
+ }
+ if s.cfg.StoreProvisioner == nil {
+ return fmt.Errorf("finish seed %s of %s: this console has no storage backend to tear its store down with", seedID, host)
+ }
+ tctx, cancel := context.WithTimeout(ctx, storeTeardownTimeout)
+ defer cancel()
+ if err := objstore.TeardownIn(tctx, s.cfg.Releases, s.cfg.StoreProvisioner, s.cfg.StoreScanner, sqid, app, env); err != nil {
+ return fmt.Errorf("tear down the store of %s: %w", host, err)
+ }
+ log.Printf("console: finish seed %s of %s/%s/%s: tore down its store (bucket %s)", seedID, sqid, app, host, storeBucketName(store))
+ return nil
+}
+
+// otherInstanceInEnvironment names an instance of app other than host
+// whose record names env, or "" when there is none. A listing it cannot
+// complete is an error: without it the finish cannot tell whether the
+// store is shared.
+func (s *Server) otherInstanceInEnvironment(ctx context.Context, sqid, app, host, env string) (string, error) {
+ recs, truncated, err := provision.ListRecords(ctx, s.cfg.Releases, sqid, app, instanceListMax)
+ if err != nil {
+ return "", fmt.Errorf("list the instances of %s/%s: %w", sqid, app, err)
+ }
+ if truncated {
+ return "", fmt.Errorf("list the instances of %s/%s: more than %d, so the finish cannot tell who uses environment %q", sqid, app, instanceListMax, env)
+ }
+ for _, r := range recs {
+ if r.Host != host && r.Environment == env {
+ return r.Host, nil
+ }
+ }
+ return "", nil
+}
+
+// deleteSeedReplica deletes every key under a --from seed's replica prefix
+// in the data bucket. The rail that makes this safe (createleftovers.go)
+// only checked a record this console can see; a record outside that
+// (another region, a pool, a fleet) was never checked, so its prefix here
+// is not known to be this seed's and is left alone.
+func (s *Server) deleteSeedReplica(ctx context.Context, rec *provision.Record) error {
+ if !s.leftoverCheckApplies(rec.Region, rec.Pool) {
+ log.Printf("console: finish seed %s of %s: its replica is outside this console's data bucket; not deleting it", rec.Seed.ID, rec.Host)
+ return nil
+ }
+ // The trailing slash keeps app.example.co from covering
+ // app.example.com's keys.
+ prefix := activator.HibernatePrefix(s.cfg.HibernatePrefix, rec.Host) + "/"
+ keys, err := s.cfg.SeedData.List(ctx, prefix)
+ if err != nil {
+ return fmt.Errorf("list the replica of %s: %w", rec.Host, err)
+ }
+ for _, key := range keys {
+ if err := s.cfg.SeedData.Delete(ctx, key); err != nil && !errors.Is(err, os.ErrNotExist) {
+ return fmt.Errorf("delete the replica of %s: %w", rec.Host, err)
+ }
+ }
+ if len(keys) > 0 {
+ log.Printf("console: finish seed %s of %s: deleted %d replica objects under %s", rec.Seed.ID, rec.Host, len(keys), prefix)
+ }
+ return nil
+}
diff --git a/internal/console/seedfinish_test.go b/internal/console/seedfinish_test.go
new file mode 100644
index 000000000..aae4a6342
--- /dev/null
+++ b/internal/console/seedfinish_test.go
@@ -0,0 +1,440 @@
+package console
+
+import (
+ "context"
+ "errors"
+ "os"
+ "path/filepath"
+ "slices"
+ "testing"
+ "time"
+
+ "github.com/carlosframework/platform/internal/dbdoor/dbdoortest"
+ "github.com/carlosframework/platform/internal/objstore"
+ "github.com/carlosframework/platform/internal/provision"
+ "github.com/carlosframework/platform/internal/release"
+ "github.com/carlosframework/platform/internal/seeds"
+)
+
+// finishFixture is the --from fixture with private storage on and one data
+// bucket, as production has it: seeds/ and every instance's replica in the
+// same store, so the leftover-replica rail and the finish see one bucket.
+func finishFixture(t *testing.T) fromFixture {
+ t.Helper()
+ f := newFromFixture(t, 3)
+ f.data = release.NewDirStore(f.root)
+ f.fx.srv.cfg.SeedData = f.data
+ enableInstances(t, f.fx, fromApp, fromManifest(true))
+ return f
+}
+
+// rollBack walks a seed the way a box does once it has been aborted: the
+// abort, then the box's CAS to rolled-back.
+func (f fromFixture) rollBack(t *testing.T, host, id string) {
+ t.Helper()
+ ctx := context.Background()
+ for _, step := range []func(*provision.Record) error{provision.SeedAbort(id), provision.SeedRolledBack(id)} {
+ if _, err := provision.SetRecordSeed(ctx, f.fx.rel, "bab", fromApp, host, step); err != nil {
+ t.Fatal(err)
+ }
+ }
+}
+
+// seedRecord writes a record of shop on host in env whose seed is id, of
+// kind, in state.
+func (f fromFixture) seedRecord(t *testing.T, host, env, id, kind, state string) {
+ t.Helper()
+ rec := provision.Record{Schema: 1, Host: host, Slug: "canary", App: fromApp, Account: "bab", Channel: "stable",
+ Environment: env, CreatedAt: time.Now().UTC(),
+ Seed: &provision.SeedSource{ID: id, Kind: kind, State: state, By: "bob@keymail.dev", At: time.Now().UTC()}}
+ if err := provision.CreateRecord(context.Background(), f.fx.rel, rec); err != nil {
+ t.Fatal(err)
+ }
+}
+
+// markedStore writes a canary store holding a grant, made by seed (""
+// for a store no seed made).
+func (f fromFixture) markedStore(t *testing.T, seed string) {
+ t.Helper()
+ at := time.Now().UTC().Add(-time.Hour)
+ putCanaryStoreOf(t, f.fx, fromApp, &objstore.Record{
+ Declared: &objstore.Declared{EnvPrefix: "S3", At: at, By: "bob@keymail.dev", Seed: seed},
+ Grant: &objstore.GrantBlock{Backend: "mem", Bucket: "carlos-bab-shop-canary", Source: objstore.GrantSourceAccountApproval, At: at, By: "operator@keymail.dev"},
+ })
+}
+
+func (f fromFixture) canaryStore(t *testing.T) *objstore.Record {
+ t.Helper()
+ rec, _, err := objstore.GetIn(context.Background(), f.fx.rel, "bab", fromApp, "canary")
+ if err != nil {
+ t.Fatal(err)
+ }
+ return rec
+}
+
+func (f fromFixture) rawKey(t *testing.T, s release.Store, key string) string {
+ t.Helper()
+ b, _, err := s.Get(context.Background(), key)
+ if err != nil {
+ t.Fatalf("read %s: %v", key, err)
+ }
+ return string(b)
+}
+
+func (f fromFixture) requireRecordGone(t *testing.T, host string) {
+ t.Helper()
+ if rec, err := f.record(t, host); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("record %s after the finish = %+v, %v; want it gone", host, rec, err)
+ }
+}
+
+func newSeedID(t *testing.T) string {
+ t.Helper()
+ id, err := seeds.NewID()
+ if err != nil {
+ t.Fatal(err)
+ }
+ return id
+}
+
+func replicaKeys(t *testing.T, f fromFixture, host string) []string {
+ t.Helper()
+ keys, err := f.data.List(context.Background(), "instances/"+host+"/")
+ if err != nil {
+ t.Fatal(err)
+ }
+ return keys
+}
+
+// The whole finish for a --from seed that failed after its first upload:
+// its own store torn down for real, its snapshot and its replica deleted,
+// and the record last. The host is then free for the same copy again: no
+// leftover store for §4 to refuse, no replica for the #199 rail to refuse.
+func TestFinishSeedTearsDownAndFreesTheHost(t *testing.T) {
+ f := finishFixture(t)
+ rec := f.copy(t, fromBody)
+ requireReady(t, rec)
+ id := rec.Seed.ID
+ store := f.canaryStore(t)
+ if store == nil || store.Declared == nil || store.Declared.Seed != id {
+ t.Fatalf("canary store = %+v; want the copy's own", store)
+ }
+ bucket := storeBucketName(store)
+ if _, ok := f.fake.Buckets[bucket]; !ok {
+ t.Fatalf("bucket %s was never made", bucket)
+ }
+ // A box placed the copy and uploaded its first replica before the
+ // abort landed (a lost publish answer, Plan 5 T5).
+ dbdoortest.Replicate(t, filepath.Join(f.root, "instances", fromHost), 2)
+ if held, err := f.fx.srv.hostHoldsReplicaData(context.Background(), fromHost); err != nil || !held {
+ t.Fatalf("replica before the finish: %v, %v", held, err)
+ }
+ sourceReplica := replicaKeys(t, f, fromSource)
+ f.rollBack(t, fromHost, id)
+
+ if err := f.fx.srv.finishSeed(context.Background(), "bab", fromApp, fromHost, id); err != nil {
+ t.Fatal(err)
+ }
+ if got := f.canaryStore(t); got != nil {
+ t.Fatalf("store record after the finish = %+v; want it gone", got)
+ }
+ if _, ok := f.fake.Buckets[bucket]; ok {
+ t.Fatalf("bucket %s survived the teardown", bucket)
+ }
+ if _, _, err := f.data.Get(context.Background(), seeds.SnapshotKey(fromHost, id)); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("snapshot after the finish: %v; want it gone", err)
+ }
+ if keys := replicaKeys(t, f, fromHost); len(keys) != 0 {
+ t.Fatalf("replica after the finish: %v", keys)
+ }
+ if held, err := f.fx.srv.hostHoldsReplicaData(context.Background(), fromHost); err != nil || held {
+ t.Fatalf("the rail after the finish: %v, %v", held, err)
+ }
+ if got := replicaKeys(t, f, fromSource); !slices.Equal(got, sourceReplica) {
+ t.Fatalf("the source's replica changed:\n before %v\n after %v", sourceReplica, got)
+ }
+ f.requireRecordGone(t, fromHost)
+
+ // Idempotent: a second finish (the sweep after an inline one) is done.
+ if err := f.fx.srv.finishSeed(context.Background(), "bab", fromApp, fromHost, id); err != nil {
+ t.Fatalf("a second finish: %v", err)
+ }
+ again := f.copy(t, fromBody)
+ requireReady(t, again)
+ if again.Seed.ID == id {
+ t.Fatal("the new copy reused the old seed id")
+ }
+ if store := f.canaryStore(t); store == nil || store.Declared == nil || store.Declared.Seed != again.Seed.ID {
+ t.Fatalf("the new copy's store = %+v; want one marked with %s", store, again.Seed.ID)
+ }
+}
+
+// A teardown that fails leaves the record rolled-back and holding the host,
+// so the next finish (the sweep's) resumes and completes.
+func TestFinishSeedResumesAfterAFailedTeardown(t *testing.T) {
+ f := finishFixture(t)
+ rec := f.copy(t, fromBody)
+ requireReady(t, rec)
+ id := rec.Seed.ID
+ f.rollBack(t, fromHost, id)
+ f.fake.FailNext["DeleteBucket"] = errors.New("the backend is down")
+
+ if err := f.fx.srv.finishSeed(context.Background(), "bab", fromApp, fromHost, id); err == nil {
+ t.Fatal("a failed teardown answered nil")
+ }
+ if got, err := f.record(t, fromHost); err != nil || got.Seed == nil || got.Seed.State != seeds.StateRolledBack {
+ t.Fatalf("record after a failed teardown = %+v, %v; want it kept, rolled-back", got, err)
+ }
+ if f.canaryStore(t) == nil {
+ t.Fatal("the store record went before its estate")
+ }
+ if err := f.fx.srv.finishSeed(context.Background(), "bab", fromApp, fromHost, id); err != nil {
+ t.Fatalf("the resumed finish: %v", err)
+ }
+ if got := f.canaryStore(t); got != nil {
+ t.Fatalf("store record after the resumed finish = %+v", got)
+ }
+ f.requireRecordGone(t, fromHost)
+}
+
+// Only a store this seed made is torn down. Any other store is left as it
+// was, and the record is still deleted.
+func TestFinishSeedLeavesAStoreItDidNotMake(t *testing.T) {
+ for name, seed := range map[string]string{
+ "another seed's": "sd_bbbbbbbbbbbbbbbbbbbbbbbbbb",
+ "no seed's": "",
+ } {
+ t.Run(name, func(t *testing.T) {
+ f := finishFixture(t)
+ id := newSeedID(t)
+ f.seedRecord(t, fromHost, "canary", id, seeds.KindImport, seeds.StateRolledBack)
+ f.markedStore(t, seed)
+ key := objstore.KeyIn("bab", fromApp, "canary")
+ before := f.rawKey(t, f.fx.rel, key)
+ if err := f.fx.srv.finishSeed(context.Background(), "bab", fromApp, fromHost, id); err != nil {
+ t.Fatal(err)
+ }
+ if after := f.rawKey(t, f.fx.rel, key); after != before {
+ t.Fatalf("the store record changed:\n before %s\n after %s", before, after)
+ }
+ f.requireRecordGone(t, fromHost)
+ })
+ }
+}
+
+// A record the finish does not own is left alone, with the store its seed
+// made: another seed, a seed not yet rolled back, no seed at all.
+func TestFinishSeedLeavesARecordItDoesNotOwn(t *testing.T) {
+ id := "sd_aaaaaaaaaaaaaaaaaaaaaaaaaa"
+ for name, c := range map[string]struct{ id, state string }{
+ "another seed": {"sd_bbbbbbbbbbbbbbbbbbbbbbbbbb", seeds.StateRolledBack},
+ "aborted, not yet": {id, seeds.StateAborted},
+ "ready": {id, seeds.StateReady},
+ "placed": {id, seeds.StatePlaced},
+ "seedless (no seed)": {"", ""},
+ } {
+ t.Run(name, func(t *testing.T) {
+ f := finishFixture(t)
+ if c.id == "" {
+ if err := provision.CreateRecord(context.Background(), f.fx.rel, provision.Record{Schema: 1, Host: fromHost, Slug: "canary",
+ App: fromApp, Account: "bab", Channel: "stable", Environment: "canary", CreatedAt: time.Now().UTC()}); err != nil {
+ t.Fatal(err)
+ }
+ } else {
+ f.seedRecord(t, fromHost, "canary", c.id, seeds.KindReplica, c.state)
+ }
+ f.markedStore(t, id)
+ if _, err := f.data.Put(context.Background(), "instances/"+fromHost+"/0000/0000000000000001-0000000000000001.ltx", []byte("x"), ""); err != nil {
+ t.Fatal(err)
+ }
+ recKey := provision.RecordKey("bab", fromApp, fromHost)
+ storeKey := objstore.KeyIn("bab", fromApp, "canary")
+ recBefore, storeBefore := f.rawKey(t, f.fx.rel, recKey), f.rawKey(t, f.fx.rel, storeKey)
+ if err := f.fx.srv.finishSeed(context.Background(), "bab", fromApp, fromHost, id); err != nil {
+ t.Fatal(err)
+ }
+ if after := f.rawKey(t, f.fx.rel, recKey); after != recBefore {
+ t.Fatalf("the record changed:\n before %s\n after %s", recBefore, after)
+ }
+ if after := f.rawKey(t, f.fx.rel, storeKey); after != storeBefore {
+ t.Fatalf("the store record changed:\n before %s\n after %s", storeBefore, after)
+ }
+ if keys := replicaKeys(t, f, fromHost); len(keys) != 1 {
+ t.Fatalf("replica = %v; want it untouched", keys)
+ }
+ })
+ }
+}
+
+// The finish decides under the store's walk lock (preflight M9). A grant
+// by another writer that held the lock while the finish queued, and took
+// the store over (clearing its seed mark), wins: the finish re-reads under
+// the lock and leaves the store standing.
+func TestFinishSeedReReadsTheStoreUnderItsLock(t *testing.T) {
+ f := finishFixture(t)
+ id := newSeedID(t)
+ f.seedRecord(t, fromHost, "canary", id, seeds.KindImport, seeds.StateRolledBack)
+ f.markedStore(t, id)
+ unlock := f.fx.srv.storeWalks.lock("bab", fromApp, "canary")
+ done := make(chan error, 1)
+ go func() { done <- f.fx.srv.finishSeed(context.Background(), "bab", fromApp, fromHost, id) }()
+ deadline := time.Now().Add(3 * time.Second)
+ for {
+ f.fx.srv.storeWalks.mu.Lock()
+ refs := 0
+ if e := f.fx.srv.storeWalks.m[storeWalkKey("bab", fromApp, "canary")]; e != nil {
+ refs = e.refs
+ }
+ f.fx.srv.storeWalks.mu.Unlock()
+ if refs == 2 {
+ break
+ }
+ if time.Now().After(deadline) {
+ unlock()
+ t.Fatalf("the finish never queued on the store lock: refs=%d", refs)
+ }
+ time.Sleep(time.Millisecond)
+ }
+ store, etag, err := objstore.GetIn(context.Background(), f.fx.rel, "bab", fromApp, "canary")
+ if err != nil {
+ t.Fatal(err)
+ }
+ store.Declared.Seed = ""
+ if _, err := objstore.PutIn(context.Background(), f.fx.rel, "bab", fromApp, "canary", store, etag); err != nil {
+ t.Fatal(err)
+ }
+ key := objstore.KeyIn("bab", fromApp, "canary")
+ taken := f.rawKey(t, f.fx.rel, key)
+ unlock()
+ if err := <-done; err != nil {
+ t.Fatal(err)
+ }
+ if after := f.rawKey(t, f.fx.rel, key); after != taken {
+ t.Fatalf("a store taken over under the lock was touched:\n before %s\n after %s", taken, after)
+ }
+ f.requireRecordGone(t, fromHost)
+}
+
+// Defence in depth (preflight M10): a store this seed made is not torn down
+// while another instance names its environment. The record still goes:
+// the store now serves the other instance, and an operator decides.
+func TestFinishSeedSparesAStoreAnotherInstanceUses(t *testing.T) {
+ f := finishFixture(t)
+ id := newSeedID(t)
+ f.seedRecord(t, fromHost, "canary", id, seeds.KindImport, seeds.StateRolledBack)
+ if err := provision.CreateRecord(context.Background(), f.fx.rel, provision.Record{Schema: 1, Host: "other.shop.example.com", Slug: "other",
+ App: fromApp, Account: "bab", Channel: "stable", Environment: "canary", CreatedAt: time.Now().UTC()}); err != nil {
+ t.Fatal(err)
+ }
+ f.markedStore(t, id)
+ key := objstore.KeyIn("bab", fromApp, "canary")
+ before := f.rawKey(t, f.fx.rel, key)
+ if err := f.fx.srv.finishSeed(context.Background(), "bab", fromApp, fromHost, id); err != nil {
+ t.Fatal(err)
+ }
+ if after := f.rawKey(t, f.fx.rel, key); after != before {
+ t.Fatalf("a shared store was touched:\n before %s\n after %s", before, after)
+ }
+ f.requireRecordGone(t, fromHost)
+}
+
+// failingRecordList fails every listing of the app's records.
+type failingRecordList struct {
+ release.Store
+ prefix string
+}
+
+func (s failingRecordList) List(ctx context.Context, prefix string) ([]string, error) {
+ if prefix == s.prefix {
+ return nil, errors.New("the bucket is unreachable")
+ }
+ return s.Store.List(ctx, prefix)
+}
+
+// When the records cannot be listed the finish cannot tell whether the
+// store is shared, so it does nothing and answers an error for the sweep
+// to retry.
+func TestFinishSeedRetriesWhenTheRecordsCannotBeListed(t *testing.T) {
+ f := finishFixture(t)
+ id := newSeedID(t)
+ f.seedRecord(t, fromHost, "canary", id, seeds.KindImport, seeds.StateRolledBack)
+ f.markedStore(t, id)
+ rel := f.fx.srv.cfg.Releases
+ f.fx.srv.cfg.Releases = failingRecordList{Store: rel, prefix: provision.RecordsPrefix("bab", fromApp)}
+ if err := f.fx.srv.finishSeed(context.Background(), "bab", fromApp, fromHost, id); err == nil {
+ t.Fatal("a failed listing answered nil")
+ }
+ if f.canaryStore(t) == nil {
+ t.Fatal("the store record went on a failed listing")
+ }
+ if _, err := f.record(t, fromHost); err != nil {
+ t.Fatalf("the record went on a failed listing: %v", err)
+ }
+}
+
+// A store this seed made with no backend to tear it down with is an error,
+// and the record keeps holding the host.
+func TestFinishSeedNeedsAStorageBackend(t *testing.T) {
+ f := finishFixture(t)
+ id := newSeedID(t)
+ f.seedRecord(t, fromHost, "canary", id, seeds.KindImport, seeds.StateRolledBack)
+ f.markedStore(t, id)
+ f.fx.srv.cfg.StoreProvisioner = nil
+ if err := f.fx.srv.finishSeed(context.Background(), "bab", fromApp, fromHost, id); err == nil {
+ t.Fatal("no backend answered nil")
+ }
+ if _, err := f.record(t, fromHost); err != nil {
+ t.Fatalf("the record went with its store still standing: %v", err)
+ }
+}
+
+// An import's finish deletes its snapshot and never the host's replica:
+// only a --from seed's replica is known to be its own.
+func TestFinishSeedLeavesAnImportsReplica(t *testing.T) {
+ f := finishFixture(t)
+ id := newSeedID(t)
+ f.seedRecord(t, fromHost, "canary", id, seeds.KindImport, seeds.StateRolledBack)
+ ctx := context.Background()
+ if _, err := f.data.Put(ctx, seeds.SnapshotKey(fromHost, id), []byte("snapshot"), ""); err != nil {
+ t.Fatal(err)
+ }
+ dbdoortest.Replicate(t, filepath.Join(f.root, "instances", fromHost), 2)
+ before := replicaKeys(t, f, fromHost)
+ if err := f.fx.srv.finishSeed(ctx, "bab", fromApp, fromHost, id); err != nil {
+ t.Fatal(err)
+ }
+ if _, _, err := f.data.Get(ctx, seeds.SnapshotKey(fromHost, id)); !errors.Is(err, os.ErrNotExist) {
+ t.Fatalf("snapshot after the finish: %v; want it gone", err)
+ }
+ if after := replicaKeys(t, f, fromHost); len(before) == 0 || !slices.Equal(after, before) {
+ t.Fatalf("an import's finish touched the replica:\n before %v\n after %v", before, after)
+ }
+ f.requireRecordGone(t, fromHost)
+}
+
+// A seed id that fails seeds.Check builds no key and deletes nothing
+// (preflight M8): every bad id would map to the one shared sentinel.
+func TestFinishSeedChecksTheSeedBeforeAnyKey(t *testing.T) {
+ f := finishFixture(t)
+ ctx := context.Background()
+ sentinel := seeds.SnapshotKey(fromHost, "sd_bad")
+ if _, err := f.data.Put(ctx, sentinel, []byte("someone else's"), ""); err != nil {
+ t.Fatal(err)
+ }
+ for _, c := range []struct{ host, id string }{{fromHost, "sd_bad"}, {"../shop", newSeedID(t)}} {
+ if err := f.fx.srv.finishSeed(ctx, "bab", fromApp, c.host, c.id); !errors.Is(err, seeds.ErrBadID) && !errors.Is(err, seeds.ErrBadHost) {
+ t.Fatalf("finishSeed(%q, %q) = %v; want a seeds.Check error", c.host, c.id, err)
+ }
+ }
+ if _, _, err := f.data.Get(ctx, sentinel); err != nil {
+ t.Fatalf("the sentinel key was deleted: %v", err)
+ }
+}
+
+// No record: nothing to finish, and no error.
+func TestFinishSeedWithNoRecord(t *testing.T) {
+ f := finishFixture(t)
+ if err := f.fx.srv.finishSeed(context.Background(), "bab", fromApp, fromHost, newSeedID(t)); err != nil {
+ t.Fatal(err)
+ }
+}