| 1 | // The console's finish for an abandoned seed (seeded-instances spec §5.6, |
| 2 | // codex R6-1). Once a box has rolled a seed back, or the delete door has |
| 3 | // moved an import no box swapped straight to rolled-back, the record still |
| 4 | // holds the hostname. finishSeed is what frees it: the store the seed made, |
| 5 | // the seed's snapshot, a --from seed's replica, and the record last. |
| 6 | |
| 7 | package console |
| 8 | |
| 9 | import ( |
| 10 | "context" |
| 11 | "errors" |
| 12 | "fmt" |
| 13 | "log" |
| 14 | "os" |
| 15 | "slices" |
| 16 | "time" |
| 17 | |
| 18 | "github.com/carlosframework/platform/internal/activator" |
| 19 | "github.com/carlosframework/platform/internal/lease" |
| 20 | "github.com/carlosframework/platform/internal/objstore" |
| 21 | "github.com/carlosframework/platform/internal/provision" |
| 22 | "github.com/carlosframework/platform/internal/release" |
| 23 | "github.com/carlosframework/platform/internal/seeds" |
| 24 | ) |
| 25 | |
| 26 | // finishSeed finishes seedID on host's record, only while that record |
| 27 | // still names seedID and says rolled-back. Any other record (another seed, |
| 28 | // another state, none) is left alone and answers nil. In order: |
| 29 | // |
| 30 | // 1. The store of the record's environment, under the store's walk lock |
| 31 | // and re-read there with the sharing check (final review F3), is torn |
| 32 | // down only while its Declared.Seed is seedID (preflight M9), and only |
| 33 | // when no other instance names that environment (M10), read strictly: |
| 34 | // a record of the app that will not decode counts as one (Plan 8 M7). |
| 35 | // A store with another seed, or none, is never touched. |
| 36 | // 2. The seed's snapshot, seeds/<host>/<id>/snapshot.db. |
| 37 | // 3. For a --from seed, every key under the host's replica prefix in the |
| 38 | // data bucket (M11). It is this seed's: the record is create-only and |
| 39 | // the create door's leftover-replica rail refused the host if anything |
| 40 | // was there, so only this seed's instance has written there since. |
| 41 | // That holds only while the record still names this seed, so the |
| 42 | // record is re-read before the listing and before every batch of |
| 43 | // deletes (review I2); and nothing is deleted while the host's lease is |
| 44 | // held, because a row may still be writing (review I1). |
| 45 | // 4. The record, by CAS on the etag of the last read that saw this seed |
| 46 | // in rolled-back. It goes last because until then it holds the |
| 47 | // hostname, so a re-import of the host never meets a leftover store. |
| 48 | // |
| 49 | // Every step is idempotent: a finish that fails part way answers an error |
| 50 | // with the record still rolled-back, and the next finish (the caller's |
| 51 | // retry, or the sweep) resumes it. seeds.Check runs before any key is |
| 52 | // built (M8). ctx bounds the whole finish; the teardown also has its own |
| 53 | // storeTeardownTimeout. Callers that finish inline on a request's behalf |
| 54 | // pass context.WithoutCancel of it, so a departing client cannot cut a |
| 55 | // teardown midway. |
| 56 | // |
| 57 | // A console with host claims armed refuses: its record deletes go through |
| 58 | // the claim (retireClaimHostRecord), and the seed doors refuse such a |
| 59 | // console anyway (seedRegionReady). |
| 60 | func (s *Server) finishSeed(ctx context.Context, sqid, app, host, seedID string) error { |
| 61 | if s.cfg.HostClaim { |
| 62 | return fmt.Errorf("finish seed %s of %s: this console has host claims armed, and finishes no seed", seedID, host) |
| 63 | } |
| 64 | if err := seeds.Check(host, seedID); err != nil { |
| 65 | return fmt.Errorf("finish seed %q of %q: %w", seedID, host, err) |
| 66 | } |
| 67 | err := s.finishSeedOnce(ctx, sqid, app, host, seedID) |
| 68 | if errors.Is(err, release.ErrCASConflict) { |
| 69 | // Another writer changed the record between the read and the |
| 70 | // delete. Every step is idempotent, so read it again once. |
| 71 | err = s.finishSeedOnce(ctx, sqid, app, host, seedID) |
| 72 | } |
| 73 | return err |
| 74 | } |
| 75 | |
| 76 | func (s *Server) finishSeedOnce(ctx context.Context, sqid, app, host, seedID string) error { |
| 77 | rec, etag, err := provision.GetRecordETag(ctx, s.cfg.Releases, sqid, app, host) |
| 78 | if errors.Is(err, os.ErrNotExist) { |
| 79 | return nil |
| 80 | } |
| 81 | if err != nil { |
| 82 | return fmt.Errorf("read the record of %s: %w", host, err) |
| 83 | } |
| 84 | if rec.Seed == nil || rec.Seed.ID != seedID || rec.Seed.State != seeds.StateRolledBack { |
| 85 | return nil |
| 86 | } |
| 87 | // Checked before anything is destroyed: without it the store would be |
| 88 | // torn down and every later finish fail with the record holding the |
| 89 | // host. |
| 90 | if s.cfg.SeedData == nil { |
| 91 | return fmt.Errorf("finish seed %s of %s: this console has no data bucket to delete its snapshot from", seedID, host) |
| 92 | } |
| 93 | if err := s.finishSeedStore(ctx, sqid, app, host, seedID, rec.Environment); err != nil { |
| 94 | return err |
| 95 | } |
| 96 | if err := s.cfg.SeedData.Delete(ctx, seeds.SnapshotKey(host, seedID)); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 97 | return fmt.Errorf("delete the snapshot of seed %s of %s: %w", seedID, host, err) |
| 98 | } |
| 99 | if rec.Seed.Kind == seeds.KindReplica { |
| 100 | var gone bool |
| 101 | if etag, gone, err = s.deleteSeedReplica(ctx, rec, etag); err != nil { |
| 102 | return err |
| 103 | } |
| 104 | if gone { |
| 105 | return nil // another finish deleted the record |
| 106 | } |
| 107 | } |
| 108 | if err := release.DeleteIfMatch(ctx, s.cfg.Releases, provision.RecordKey(sqid, app, host), etag); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 109 | return fmt.Errorf("delete the record of %s: %w", host, err) |
| 110 | } |
| 111 | log.Printf("console: finished seed %s of %s/%s/%s", seedID, sqid, app, host) |
| 112 | s.bumpSerial(ctx) |
| 113 | return nil |
| 114 | } |
| 115 | |
| 116 | // finishSeedStore is the finish's first step: the store of env, torn down |
| 117 | // only while seedID made it and no other instance uses it. A first look |
| 118 | // runs without the lock, and a store it keeps ends the step there. The |
| 119 | // teardown itself runs under the store's walk lock, after the store record |
| 120 | // and the sharing check are read again there (final review F3): a grant (a |
| 121 | // begin's, or carlos store create --environment) and an instance's |
| 122 | // environment binding (seedenvbind.go) take that lock too, and a binding |
| 123 | // clears the seed's mark, so whatever bound since the first look keeps the |
| 124 | // store. |
| 125 | func (s *Server) finishSeedStore(ctx context.Context, sqid, app, host, seedID, env string) error { |
| 126 | store, err := s.seedStoreToTear(ctx, sqid, app, host, seedID, env) |
| 127 | if store == nil || err != nil { |
| 128 | return err |
| 129 | } |
| 130 | if finishSeedCheckedHook != nil { |
| 131 | finishSeedCheckedHook(host) |
| 132 | } |
| 133 | defer s.storeWalks.lock(sqid, app, env)() |
| 134 | if store, err = s.seedStoreToTear(ctx, sqid, app, host, seedID, env); store == nil || err != nil { |
| 135 | return err |
| 136 | } |
| 137 | if s.cfg.StoreProvisioner == nil { |
| 138 | return fmt.Errorf("finish seed %s of %s: this console has no storage backend to tear its store down with", seedID, host) |
| 139 | } |
| 140 | tctx, cancel := context.WithTimeout(ctx, storeTeardownTimeout) |
| 141 | defer cancel() |
| 142 | if err := objstore.TeardownIn(tctx, s.cfg.Releases, s.cfg.StoreProvisioner, s.cfg.StoreScanner, sqid, app, env); err != nil { |
| 143 | return fmt.Errorf("tear down the store of %s: %w", host, err) |
| 144 | } |
| 145 | log.Printf("console: finish seed %s of %s/%s/%s: tore down its store (bucket %s)", seedID, sqid, app, host, storeBucketName(store)) |
| 146 | return nil |
| 147 | } |
| 148 | |
| 149 | // finishSeedCheckedHook, when set (tests), runs between the finish's first |
| 150 | // look at the store and its locked re-read. |
| 151 | var finishSeedCheckedHook func(host string) |
| 152 | |
| 153 | // seedStoreToTear reads env's store and answers it when seedID made it and |
| 154 | // no other instance names env; nil when the store is to be kept (logged |
| 155 | // for an operator when another instance may use it). |
| 156 | func (s *Server) seedStoreToTear(ctx context.Context, sqid, app, host, seedID, env string) (*objstore.Record, error) { |
| 157 | store, _, err := objstore.GetIn(ctx, s.cfg.Releases, sqid, app, env) |
| 158 | if err != nil { |
| 159 | return nil, fmt.Errorf("read the store of %s: %w", host, err) |
| 160 | } |
| 161 | if store == nil || store.Declared == nil || store.Declared.Seed != seedID { |
| 162 | return nil, nil |
| 163 | } |
| 164 | other, unread, err := s.otherInstanceInEnvironment(ctx, sqid, app, host, env) |
| 165 | if err != nil { |
| 166 | return nil, err |
| 167 | } |
| 168 | if unread { |
| 169 | log.Printf("console: finish seed %s of %s/%s/%s: OPERATOR: the record of instance %s could not be read, so it may use environment %q, and the store seed %s made (bucket %s) is left standing; tear it down by hand if it is not wanted", |
| 170 | seedID, sqid, app, host, other, env, seedID, storeBucketName(store)) |
| 171 | return nil, nil |
| 172 | } |
| 173 | if other != "" { |
| 174 | log.Printf("console: finish seed %s of %s/%s/%s: OPERATOR: instance %s also uses environment %q, so the store seed %s made (bucket %s) is left standing; tear it down by hand if it is not wanted", |
| 175 | seedID, sqid, app, host, other, env, seedID, storeBucketName(store)) |
| 176 | return nil, nil |
| 177 | } |
| 178 | return store, nil |
| 179 | } |
| 180 | |
| 181 | // otherInstanceInEnvironment names an instance of app other than host |
| 182 | // whose record names env, or "" when there is none. It reads strictly |
| 183 | // (provision.ListRecordsStrict, preflight M7): it runs unattended from the |
| 184 | // seeds sweep, and its "none" tears a store down. A listing it cannot |
| 185 | // complete, or one with more than instanceListMax records, is an error: |
| 186 | // the finish cannot tell whether the store is shared, so the record stays |
| 187 | // rolled-back for a later finish. A record that reads but will not decode, |
| 188 | // or disagrees with its path, counts as sharing: it is named, with unread |
| 189 | // true, since what it says about env is not known. |
| 190 | func (s *Server) otherInstanceInEnvironment(ctx context.Context, sqid, app, host, env string) (other string, unread bool, err error) { |
| 191 | recs, unreadable, err := provision.ListRecordsStrict(ctx, s.cfg.Releases, sqid, app, instanceListMax) |
| 192 | if errors.Is(err, provision.ErrRecordsTruncated) { |
| 193 | return "", false, fmt.Errorf("list the instances of %s/%s: more than %d, so the finish cannot tell who uses environment %q", sqid, app, instanceListMax, env) |
| 194 | } |
| 195 | if err != nil { |
| 196 | return "", false, fmt.Errorf("list the instances of %s/%s: %w", sqid, app, err) |
| 197 | } |
| 198 | if len(unreadable) > 0 { |
| 199 | hosts := make([]string, 0, len(unreadable)) |
| 200 | for h := range unreadable { |
| 201 | hosts = append(hosts, h) |
| 202 | } |
| 203 | slices.Sort(hosts) |
| 204 | return hosts[0], true, nil |
| 205 | } |
| 206 | for _, r := range recs { |
| 207 | if r.Host != host && r.Environment == env { |
| 208 | return r.Host, false, nil |
| 209 | } |
| 210 | } |
| 211 | return "", false, nil |
| 212 | } |
| 213 | |
| 214 | // seedReplicaBatch is how many replica keys are deleted between two |
| 215 | // re-reads of the record. |
| 216 | const seedReplicaBatch = 100 |
| 217 | |
| 218 | // deleteSeedReplica deletes every key under a --from seed's replica prefix |
| 219 | // in the data bucket, and answers the etag the record delete is to be |
| 220 | // conditional on (the last read that saw this seed rolled-back), or gone |
| 221 | // when the record was deleted meanwhile (another finish). |
| 222 | // |
| 223 | // The rail that makes the delete safe (createleftovers.go) only checked a |
| 224 | // record this console can see; a record outside that (another region, a |
| 225 | // pool, a fleet) was never checked, so its prefix here is not known to be |
| 226 | // this seed's and is left alone. |
| 227 | // |
| 228 | // Nothing is deleted while the host's lease (lock.json under the prefix) is |
| 229 | // held by the lease package's own rule (lease.LockState): a row this seed |
| 230 | // started may still be running and writing there. A lease that is absent, |
| 231 | // released or expired past its margin does not block, so a box that died |
| 232 | // does not hold the host for ever. A held lease is an error, and the |
| 233 | // record stays rolled-back for the next finish. |
| 234 | func (s *Server) deleteSeedReplica(ctx context.Context, rec *provision.Record, etag string) (string, bool, error) { |
| 235 | sqid, app, host, seedID := rec.Account, rec.App, rec.Host, rec.Seed.ID |
| 236 | if !s.leftoverCheckApplies(rec.Region, rec.Pool) { |
| 237 | log.Printf("console: finish seed %s of %s: its replica is outside this console's data bucket; not deleting it", seedID, host) |
| 238 | return etag, false, nil |
| 239 | } |
| 240 | // The read before the teardown can be a minute old: the host may have |
| 241 | // been finished and created again since, by a new instance that is |
| 242 | // writing here now. |
| 243 | etag, gone, err := s.seedRecordETag(ctx, sqid, app, host, seedID) |
| 244 | if gone || err != nil { |
| 245 | return "", gone, err |
| 246 | } |
| 247 | location := activator.HibernatePrefix(s.cfg.HibernatePrefix, host) |
| 248 | held, owner, expires, err := lease.LockState(ctx, s.cfg.HibernateBucket, location, s.cfg.HibernateRegion) |
| 249 | if err != nil { |
| 250 | return "", false, fmt.Errorf("read the lease of %s: %w", host, err) |
| 251 | } |
| 252 | if held { |
| 253 | return "", false, fmt.Errorf("finish seed %s of %s: its lease is held by %q until %s, so its replica is not deleted yet", |
| 254 | seedID, host, owner, expires.UTC().Format(time.RFC3339)) |
| 255 | } |
| 256 | // The trailing slash keeps app.example.co from covering |
| 257 | // app.example.com's keys. |
| 258 | prefix := location + "/" |
| 259 | keys, err := s.cfg.SeedData.List(ctx, prefix) |
| 260 | if err != nil { |
| 261 | return "", false, fmt.Errorf("list the replica of %s: %w", host, err) |
| 262 | } |
| 263 | for start := 0; start < len(keys); start += seedReplicaBatch { |
| 264 | now, gone, err := s.seedRecordETag(ctx, sqid, app, host, seedID) |
| 265 | if gone || err != nil { |
| 266 | return "", gone, err |
| 267 | } |
| 268 | if now != etag { |
| 269 | return "", false, fmt.Errorf("finish seed %s of %s: the record changed while its replica was being deleted", seedID, host) |
| 270 | } |
| 271 | for _, key := range keys[start:min(start+seedReplicaBatch, len(keys))] { |
| 272 | if err := s.cfg.SeedData.Delete(ctx, key); err != nil && !errors.Is(err, os.ErrNotExist) { |
| 273 | return "", false, fmt.Errorf("delete the replica of %s: %w", host, err) |
| 274 | } |
| 275 | } |
| 276 | } |
| 277 | if len(keys) > 0 { |
| 278 | log.Printf("console: finish seed %s of %s: deleted %d replica objects under %s", seedID, host, len(keys), prefix) |
| 279 | } |
| 280 | return etag, false, nil |
| 281 | } |
| 282 | |
| 283 | // seedRecordETag re-reads host's record and answers its etag while it |
| 284 | // still names seedID in rolled-back, or gone when there is no record. A |
| 285 | // record that names anything else is an error, not "done": the finish only |
| 286 | // says done when the seed's own record is gone. |
| 287 | func (s *Server) seedRecordETag(ctx context.Context, sqid, app, host, seedID string) (etag string, gone bool, err error) { |
| 288 | rec, etag, err := provision.GetRecordETag(ctx, s.cfg.Releases, sqid, app, host) |
| 289 | switch { |
| 290 | case errors.Is(err, os.ErrNotExist): |
| 291 | return "", true, nil |
| 292 | case err != nil: |
| 293 | return "", false, fmt.Errorf("read the record of %s: %w", host, err) |
| 294 | case rec.Seed == nil || rec.Seed.ID != seedID || rec.Seed.State != seeds.StateRolledBack: |
| 295 | return "", false, fmt.Errorf("finish seed %s of %s: the record no longer names this seed rolled-back", seedID, host) |
| 296 | } |
| 297 | return etag, false, nil |
| 298 | } |
| 299 | |
amadan