| 1 | package console |
| 2 | |
| 3 | import ( |
| 4 | "bytes" |
| 5 | "context" |
| 6 | "encoding/json" |
| 7 | "errors" |
| 8 | "fmt" |
| 9 | "os" |
| 10 | "time" |
| 11 | |
| 12 | "github.com/carlosframework/platform/internal/dnsrec" |
| 13 | "github.com/carlosframework/platform/internal/domains" |
| 14 | "github.com/carlosframework/platform/internal/hostops" |
| 15 | "github.com/carlosframework/platform/internal/lease" |
| 16 | "github.com/carlosframework/platform/internal/mail" |
| 17 | "github.com/carlosframework/platform/internal/objstore" |
| 18 | "github.com/carlosframework/platform/internal/release" |
| 19 | ) |
| 20 | |
| 21 | type ClaimPurgeCoordinator struct { |
| 22 | Server *Server |
| 23 | Controller hostops.Controller |
| 24 | Authority *hostops.ProductionAuthority |
| 25 | Versions lease.VersionedStore |
| 26 | ControlBucket string |
| 27 | Owner hostops.RegistrationOwner |
| 28 | } |
| 29 | |
| 30 | func (p *ClaimPurgeCoordinator) control() (*hostops.ControlWriter, error) { |
| 31 | if p.Controller.Control == nil || p.Authority == nil || p.Versions == nil || p.ControlBucket == "" { |
| 32 | return nil, hostops.ErrContract |
| 33 | } |
| 34 | return p.Controller.Control, nil |
| 35 | } |
| 36 | |
| 37 | // stage runs one purge stage as a durable effect under the purge operation, |
| 38 | // so every bucket write it makes passes the data boundary on that permit. |
| 39 | // A stage that returns ErrPending joined cleanly and is retried later. |
| 40 | func (p *ClaimPurgeCoordinator) stage(ctx context.Context, h *hostops.OperationHandle, name string, fn func(context.Context) error) error { |
| 41 | id, err := claimEffectID() |
| 42 | if err != nil { |
| 43 | return err |
| 44 | } |
| 45 | var callErr error |
| 46 | _, err = h.Runner(p.Controller.Now).Run(ctx, hostops.Effect{ID: id, Resource: "purge-stage:" + name, Action: name}, func(ctx context.Context) (string, error) { |
| 47 | observed, issued := hostops.ObserveAppMutationEffects(ctx) |
| 48 | callErr = fn(observed) |
| 49 | if callErr == nil { |
| 50 | return "joined", nil |
| 51 | } |
| 52 | if !issued() { |
| 53 | return "", errors.Join(hostops.ErrNotApplied, callErr) |
| 54 | } |
| 55 | // Only the plain sentinel says the joined stage needs another pass. |
| 56 | // A joined transport error or a nested not-applied result cannot clear |
| 57 | // ownership of an effect this outer stage already issued. |
| 58 | if callErr == hostops.ErrPending { |
| 59 | return "joined", nil |
| 60 | } |
| 61 | return "", fmt.Errorf("purge stage %s returned after an issued effect: %v", name, callErr) |
| 62 | }) |
| 63 | return errors.Join(callErr, err) |
| 64 | } |
| 65 | |
| 66 | // localSQLStage is restricted to the two joined SQL projections. Their |
| 67 | // failures remain retryable without inventing an uncertain remote effect. |
| 68 | func (p *ClaimPurgeCoordinator) localSQLStage(ctx context.Context, h *hostops.OperationHandle, name string, fn func(context.Context) error) error { |
| 69 | if name != "bind-operation" && name != "app-row" { |
| 70 | return hostops.ErrContract |
| 71 | } |
| 72 | var localErr error |
| 73 | err := p.stage(ctx, h, name, func(ctx context.Context) error { localErr = fn(ctx); return nil }) |
| 74 | return errors.Join(localErr, err) |
| 75 | } |
| 76 | |
| 77 | func (p *ClaimPurgeCoordinator) Request(ctx context.Context, account, app, id, reason, by string) (string, error) { |
| 78 | s := p.Server |
| 79 | w, err := p.control() |
| 80 | if err != nil { |
| 81 | return "", err |
| 82 | } |
| 83 | acct, err := s.store.AccountBySqid(account) |
| 84 | if err != nil { |
| 85 | return "", err |
| 86 | } |
| 87 | if acct == nil { |
| 88 | return "", hostops.ErrIdentity |
| 89 | } |
| 90 | journalKey, err := hostops.PurgeKey(account, app) |
| 91 | if err != nil { |
| 92 | return "", err |
| 93 | } |
| 94 | existingRaw, _, err := s.cfg.Releases.Get(ctx, journalKey) |
| 95 | if err == nil { |
| 96 | var existing hostops.PurgeJournal |
| 97 | if err = json.Unmarshal(existingRaw, &existing); err != nil { |
| 98 | return "", err |
| 99 | } |
| 100 | return existing.ID, p.Resume(ctx, account, app, existing.ID, existing.AppID) |
| 101 | } |
| 102 | if !errors.Is(err, os.ErrNotExist) { |
| 103 | return "", err |
| 104 | } |
| 105 | appID, err := s.store.ClaimAppID(acct.ID, app) |
| 106 | if err != nil { |
| 107 | return "", err |
| 108 | } |
| 109 | admission, err := hostops.ReadAdmission(ctx, s.cfg.Releases, account, app, appID) |
| 110 | if err != nil { |
| 111 | return "", err |
| 112 | } |
| 113 | if admission.Value.LastPurge != nil && admission.Value.LastPurge.ID == id { |
| 114 | return id, nil |
| 115 | } |
| 116 | if id == "" || (reason != "owner" && reason != "automatic") { |
| 117 | return "", hostops.ErrIdentity |
| 118 | } |
| 119 | control, _, err := s.store.AppFlags(acct.ID, app) |
| 120 | if err != nil { |
| 121 | return "", err |
| 122 | } |
| 123 | if control { |
| 124 | return "", hostops.ErrPending |
| 125 | } |
| 126 | if err = release.PurgeReady(ctx, s.cfg.Releases, s.cfg.ReleasePub, account, app); err != nil { |
| 127 | return "", err |
| 128 | } |
| 129 | tomb, err := release.ReadAppTombstone(ctx, s.cfg.Releases, account, app) |
| 130 | if err != nil { |
| 131 | return "", err |
| 132 | } |
| 133 | if tomb == nil { |
| 134 | return "", release.ErrNotTrashed |
| 135 | } |
| 136 | if reason == "automatic" && p.Controller.Now().Before(tomb.DeletedAt.Add(release.TrashCooloff)) { |
| 137 | return "", hostops.ErrPending |
| 138 | } |
| 139 | key := release.AppTombstoneKey(account, app) |
| 140 | body, tag, err := s.cfg.Releases.Get(ctx, key) |
| 141 | if err != nil { |
| 142 | return "", err |
| 143 | } |
| 144 | // Backend inventory/readiness precedes every destructive action. |
| 145 | effects := map[string]json.RawMessage{} |
| 146 | envs, err := objstore.StrictStoreEnvironments(ctx, s.cfg.Releases, account, app) |
| 147 | if err != nil { |
| 148 | return "", err |
| 149 | } |
| 150 | for _, env := range append(envs, "") { |
| 151 | rec, _, err := objstore.StrictGetIn(ctx, s.cfg.Releases, account, app, env) |
| 152 | if err != nil { |
| 153 | return "", err |
| 154 | } |
| 155 | if rec != nil && (rec.Provisioned != nil || rec.Grant != nil) && s.cfg.StoreProvisioner == nil { |
| 156 | return "", hostops.ErrContract |
| 157 | } |
| 158 | raw, err := json.Marshal(rec) |
| 159 | if err != nil { |
| 160 | return "", err |
| 161 | } |
| 162 | effects["store."+env] = raw |
| 163 | } |
| 164 | mrec, _, err := mail.StrictGet(ctx, s.cfg.Releases, account, app) |
| 165 | if err != nil { |
| 166 | return "", err |
| 167 | } |
| 168 | mailDomains, err := s.mailAppDomains(ctx, acct, app) |
| 169 | if err != nil { |
| 170 | return "", err |
| 171 | } |
| 172 | if s.cfg.MailProvisioner == nil && ((mrec != nil && (mrec.Provisioned != nil || len(mrec.Standalone) > 0)) || len(mailDomains) > 0) { |
| 173 | return "", hostops.ErrContract |
| 174 | } |
| 175 | if err = p.mailReady(ctx, account, app); err != nil { |
| 176 | return "", err |
| 177 | } |
| 178 | j := hostops.PurgeJournal{V: 1, Purge: hostops.Purge{ID: id, Phase: "reserved"}, AppID: appID, Account: account, App: app, |
| 179 | RequestedBy: by, RequestedAt: p.Controller.Now().UTC().Format(time.RFC3339Nano), Reason: reason, AdmissionEpoch: admission.Value.Epoch, |
| 180 | Tombstone: hostops.PurgeObject{Key: key, ETag: tag, Body: json.RawMessage(body)}, Hosts: map[string]hostops.PurgeHost{}, Effects: effects} |
| 181 | // Registered reservation, then create-only journal: both typed. |
| 182 | if _, err = w.ReservePurge(ctx, j, p.Owner, p.Controller.Now()); err != nil { |
| 183 | return "", err |
| 184 | } |
| 185 | return id, p.Resume(ctx, account, app, id, appID) |
| 186 | } |
| 187 | |
| 188 | func (p *ClaimPurgeCoordinator) Resume(ctx context.Context, account, app, id, appID string) error { |
| 189 | s := p.Server |
| 190 | w, err := p.control() |
| 191 | if err != nil { |
| 192 | return err |
| 193 | } |
| 194 | j, _, err := hostops.ReadPurgeJournal(ctx, s.cfg.Releases, account, app, id, appID) |
| 195 | if errors.Is(err, os.ErrNotExist) { |
| 196 | // Journal gone: only a completed purge's tail (request discard). |
| 197 | return w.RemovePurgeJournal(ctx, account, app, id) |
| 198 | } |
| 199 | if err != nil { |
| 200 | return err |
| 201 | } |
| 202 | acct, err := s.store.AccountBySqid(account) |
| 203 | if err != nil { |
| 204 | return err |
| 205 | } |
| 206 | if acct == nil { |
| 207 | return hostops.ErrIdentity |
| 208 | } |
| 209 | a, err := hostops.ReadAdmission(ctx, s.cfg.Releases, account, app, appID) |
| 210 | if err != nil { |
| 211 | return err |
| 212 | } |
| 213 | if j.Phase == "done" && a.Value.Op == nil { |
| 214 | return w.RemovePurgeJournal(ctx, account, app, id) |
| 215 | } |
| 216 | // BeginPurge transfers a reserved journal or replays the existing op. |
| 217 | h, err := w.BeginPurge(ctx, account, app, appID, id, p.Owner) |
| 218 | if err != nil { |
| 219 | return err |
| 220 | } |
| 221 | // Every claim write in this pass (host claims, fences) carries the purge |
| 222 | // operation's lifecycle grant. |
| 223 | ctx = h.Lifecycle(ctx) |
| 224 | if j.Phase == "done" { |
| 225 | return p.Controller.CompletePurgeFences(ctx, h) |
| 226 | } |
| 227 | scope := claimSQLScope{AccountID: acct.ID, App: app, AppID: appID} |
| 228 | if j.Phase != "retired" { |
| 229 | if a, err = hostops.ReadAdmission(ctx, s.cfg.Releases, account, app, appID); err != nil { |
| 230 | return err |
| 231 | } |
| 232 | revision := a.Value.Revision |
| 233 | if err = p.localSQLStage(ctx, h, "bind-operation", func(ctx context.Context) error { |
| 234 | return s.store.WithAppMutation(ctx, []sqlGrant{permitGrant(ctx, s.cfg.Releases, account, scope)}, func(st *Store) error { |
| 235 | return st.BindClaimAppOperation(acct.ID, app, appID, id, revision) |
| 236 | }) |
| 237 | }); err != nil { |
| 238 | return err |
| 239 | } |
| 240 | } |
| 241 | switch j.Phase { |
| 242 | case "reserved": |
| 243 | body, tag, err := s.cfg.Releases.Get(ctx, j.Tombstone.Key) |
| 244 | if err != nil { |
| 245 | return err |
| 246 | } |
| 247 | if tag != j.Tombstone.ETag || !bytes.Equal(body, j.Tombstone.Body) { |
| 248 | return hostops.ErrPending |
| 249 | } |
| 250 | if err = release.PurgeReady(ctx, s.cfg.Releases, s.cfg.ReleasePub, account, app); err != nil { |
| 251 | return err |
| 252 | } |
| 253 | protected, _, err := s.store.AppFlags(acct.ID, app) |
| 254 | if err != nil { |
| 255 | return err |
| 256 | } |
| 257 | if protected { |
| 258 | return hostops.ErrPending |
| 259 | } |
| 260 | if err = w.DrainPurgeRegistrations(ctx, account, app, appID, id, p.Controller.Now(), p.Controller.RecoverCreate); err != nil { |
| 261 | return err |
| 262 | } |
| 263 | // The census is keyed by this purge's reservation: until every |
| 264 | // roster box has published its report for it, the owned purge is |
| 265 | // waiting (journaled, admission closed), not failed. |
| 266 | census, err := p.Authority.Census(ctx, account, app, id) |
| 267 | if errors.Is(err, os.ErrNotExist) { |
| 268 | return errors.Join(hostops.ErrPending, err) |
| 269 | } |
| 270 | if err != nil { |
| 271 | return err |
| 272 | } |
| 273 | records := map[string]bool{} |
| 274 | for _, r := range census.Records { |
| 275 | records[r.Holder.Host] = true |
| 276 | } |
| 277 | for _, report := range census.Reports { |
| 278 | for _, installation := range report.Installations { |
| 279 | if installation.Account == account && installation.App == app && !records[installation.Host] { |
| 280 | return hostops.ErrPending |
| 281 | } |
| 282 | } |
| 283 | } |
| 284 | if err = p.captureDomains(ctx, account, app, id, appID); err != nil { |
| 285 | return err |
| 286 | } |
| 287 | if err = p.Controller.CensusPurge(ctx, account, app, id, appID); err != nil { |
| 288 | return err |
| 289 | } |
| 290 | return hostops.ErrPending |
| 291 | case "adopted": |
| 292 | return p.Controller.ClaimPurgeHosts(ctx, account, app, id, appID) |
| 293 | case "claimed": |
| 294 | if err = p.Controller.PurgeBarrier(ctx, account, app, id, appID); err != nil { |
| 295 | return err |
| 296 | } |
| 297 | scope := mail.ClaimMailScope{Account: account, App: app, AppID: appID, Operation: id} |
| 298 | if err = p.stage(ctx, h, "teardown", func(ctx context.Context) error { |
| 299 | return mail.WithClaimMailResource(mail.WithClaimMailScope(ctx, scope), s.claimMailStore(), func(held context.Context) error { |
| 300 | return p.teardown(held, h, account, app, id, appID) |
| 301 | }) |
| 302 | }); err != nil { |
| 303 | return err |
| 304 | } |
| 305 | fresh, tag, err := hostops.ReadPurgeJournal(ctx, s.cfg.Releases, account, app, id, appID) |
| 306 | if err != nil { |
| 307 | return err |
| 308 | } |
| 309 | if fresh.Phase != "claimed" { |
| 310 | return hostops.ErrPending |
| 311 | } |
| 312 | fresh.Phase = "torn-down" |
| 313 | _, err = w.SavePurgeJournal(ctx, fresh, tag) |
| 314 | return err |
| 315 | case "torn-down": |
| 316 | if err = p.stage(ctx, h, "retire-hosts", func(ctx context.Context) error { |
| 317 | return p.Controller.RetirePurgeHosts(ctx, account, app, id, appID) |
| 318 | }); err != nil { |
| 319 | return err |
| 320 | } |
| 321 | if err = p.stage(ctx, h, "retire-domains", func(ctx context.Context) error { |
| 322 | return p.retireDomains(ctx, account, app, id, appID) |
| 323 | }); err != nil { |
| 324 | return err |
| 325 | } |
| 326 | fresh, tag, err := hostops.ReadPurgeJournal(ctx, s.cfg.Releases, account, app, id, appID) |
| 327 | if err != nil { |
| 328 | return err |
| 329 | } |
| 330 | fresh.Phase = "retired" |
| 331 | _, err = w.SavePurgeJournal(ctx, fresh, tag) |
| 332 | return err |
| 333 | case "retired": |
| 334 | default: |
| 335 | return hostops.ErrPending |
| 336 | } |
| 337 | if err = p.stage(ctx, h, "sweep", func(ctx context.Context) error { |
| 338 | if err := hostops.PreparePurgeSweep(ctx, s.cfg.Releases, w, p.Versions, p.ControlBucket, account, app, id, appID); err != nil { |
| 339 | return err |
| 340 | } |
| 341 | if err := hostops.DeletePurgeSweep(ctx, s.cfg.Releases, w, p.Versions, p.ControlBucket, account, app, id, appID); err != nil { |
| 342 | return err |
| 343 | } |
| 344 | return hostops.VerifyPurgeSweep(ctx, s.cfg.Releases, w, p.Versions, p.ControlBucket, account, app, id, appID) |
| 345 | }); err != nil { |
| 346 | return err |
| 347 | } |
| 348 | j, tag, err := hostops.ReadPurgeJournal(ctx, s.cfg.Releases, account, app, id, appID) |
| 349 | if err != nil { |
| 350 | return err |
| 351 | } |
| 352 | if !j.Cleanup.Tombstone { |
| 353 | if err = p.stage(ctx, h, "tombstone", func(ctx context.Context) error { |
| 354 | err := release.DeleteIfMatch(ctx, s.cfg.Releases, j.Tombstone.Key, j.Tombstone.ETag) |
| 355 | if err != nil { |
| 356 | if _, _, rerr := s.cfg.Releases.Get(ctx, j.Tombstone.Key); !errors.Is(rerr, os.ErrNotExist) { |
| 357 | return errors.Join(err, rerr) |
| 358 | } |
| 359 | } |
| 360 | return nil |
| 361 | }); err != nil { |
| 362 | return err |
| 363 | } |
| 364 | j.Cleanup.Tombstone = true |
| 365 | if tag, err = w.SavePurgeJournal(ctx, j, tag); err != nil { |
| 366 | return err |
| 367 | } |
| 368 | } |
| 369 | if !j.Cleanup.AppRow { |
| 370 | // The SQL boundary admits this delete on the stage's permit. |
| 371 | if err = p.localSQLStage(ctx, h, "app-row", func(ctx context.Context) error { |
| 372 | return s.store.WithAppMutation(ctx, []sqlGrant{permitGrant(ctx, s.cfg.Releases, account, scope)}, func(st *Store) error { |
| 373 | if err := st.MarkClaimAppRetired(acct.ID, app, appID, p.Controller.Now()); err != nil { |
| 374 | return err |
| 375 | } |
| 376 | return st.PurgeClaimAppRow(ctx, acct.ID, app, appID, id) |
| 377 | }) |
| 378 | }); err != nil { |
| 379 | return err |
| 380 | } |
| 381 | j.Cleanup.AppRow = true |
| 382 | if tag, err = w.SavePurgeJournal(ctx, j, tag); err != nil { |
| 383 | return err |
| 384 | } |
| 385 | } |
| 386 | // The purge retired its instances' incarnations (retire-hosts); their |
| 387 | // move log outbox rows go with them, before the purge is done. |
| 388 | if err = s.purgeMoveLogOfPurgedHosts(j); err != nil { |
| 389 | return err |
| 390 | } |
| 391 | j.Phase = "done" |
| 392 | if _, err = w.SavePurgeJournal(ctx, j, tag); err != nil { |
| 393 | return err |
| 394 | } |
| 395 | return p.Controller.CompletePurgeFences(ctx, h) |
| 396 | } |
| 397 | |
| 398 | // purgeMoveLogOfPurgedHosts drops the move log outbox rows of every instance |
| 399 | // incarnation the purge journal records (Plan 6 Task 3). It is local and |
| 400 | // idempotent, so a replayed stage repeats it harmlessly. |
| 401 | func (s *Server) purgeMoveLogOfPurgedHosts(j hostops.PurgeJournal) error { |
| 402 | for host, ph := range j.Hosts { |
| 403 | if ph.Holder == nil || ph.Holder.Incarnation == "" { |
| 404 | continue |
| 405 | } |
| 406 | if err := s.store.PurgeMoveLogIncarnation(host, ph.Holder.Incarnation); err != nil { |
| 407 | return err |
| 408 | } |
| 409 | } |
| 410 | return nil |
| 411 | } |
| 412 | |
| 413 | func (p *ClaimPurgeCoordinator) SweepRequests(ctx context.Context) error { |
| 414 | w, err := p.control() |
| 415 | if err != nil { |
| 416 | return err |
| 417 | } |
| 418 | apps, err := hostops.Apps(ctx, p.Controller.Store) |
| 419 | if err != nil { |
| 420 | return err |
| 421 | } |
| 422 | for _, app := range apps { |
| 423 | key, err := hostops.PurgeRequestKey(app.Account, app.Name) |
| 424 | if err != nil { |
| 425 | return err |
| 426 | } |
| 427 | raw, _, err := p.Controller.Store.Get(ctx, key) |
| 428 | if errors.Is(err, os.ErrNotExist) { |
| 429 | continue |
| 430 | } |
| 431 | if err != nil { |
| 432 | return err |
| 433 | } |
| 434 | removed, err := w.DiscardPurgeRequest(ctx, app.Account, app.Name) |
| 435 | if err != nil { |
| 436 | return err |
| 437 | } |
| 438 | if removed { |
| 439 | continue |
| 440 | } |
| 441 | var request release.ClaimPurgeRequest |
| 442 | if err = json.Unmarshal(raw, &request); err != nil { |
| 443 | return err |
| 444 | } |
| 445 | if request.V != 1 || request.ID == "" || request.AppID == "" || request.Account != app.Account || request.App != app.Name || request.TombstoneKey != release.AppTombstoneKey(app.Account, app.Name) { |
| 446 | return hostops.ErrIdentity |
| 447 | } |
| 448 | // An existing reservation resumes with its captured marker. |
| 449 | existing, _, perr := hostops.ReadPurge(ctx, p.Controller.Store, app.Account, app.Name) |
| 450 | if perr == nil { |
| 451 | if err = p.Resume(ctx, app.Account, app.Name, existing.ID, request.AppID); err != nil && !errors.Is(err, hostops.ErrPending) { |
| 452 | return err |
| 453 | } |
| 454 | continue |
| 455 | } |
| 456 | if !errors.Is(perr, os.ErrNotExist) { |
| 457 | return perr |
| 458 | } |
| 459 | body, tag, err := p.Controller.Store.Get(ctx, request.TombstoneKey) |
| 460 | if err != nil { |
| 461 | return err |
| 462 | } |
| 463 | if tag != request.TombstoneETag || !bytes.Equal(body, request.Tombstone) { |
| 464 | return hostops.ErrPending |
| 465 | } |
| 466 | acct, err := p.Server.store.AccountBySqid(app.Account) |
| 467 | if err != nil { |
| 468 | return err |
| 469 | } |
| 470 | if acct == nil { |
| 471 | return hostops.ErrIdentity |
| 472 | } |
| 473 | appID, err := p.Server.store.ClaimAppID(acct.ID, app.Name) |
| 474 | if err != nil { |
| 475 | return err |
| 476 | } |
| 477 | if appID != request.AppID { |
| 478 | return hostops.ErrIdentity |
| 479 | } |
| 480 | if _, err = p.Request(ctx, app.Account, app.Name, request.ID, "automatic", "auto-purge"); err != nil && !errors.Is(err, hostops.ErrPending) { |
| 481 | return err |
| 482 | } |
| 483 | } |
| 484 | return nil |
| 485 | } |
| 486 | |
| 487 | func (p *ClaimPurgeCoordinator) mailReady(ctx context.Context, account, app string) error { |
| 488 | estate := false |
| 489 | record, _, err := mail.StrictGet(ctx, p.Controller.Store, account, app) |
| 490 | if err != nil { |
| 491 | return err |
| 492 | } |
| 493 | if record != nil && (record.Provisioned != nil || len(record.Standalone) != 0) { |
| 494 | estate = true |
| 495 | } |
| 496 | // Complete inventories (C6): an unreadable record or claim stops the purge |
| 497 | // instead of being skipped and outliving it. |
| 498 | domainRecords, err := mail.StrictListDomains(ctx, p.Controller.Store, account) |
| 499 | if err != nil { |
| 500 | return err |
| 501 | } |
| 502 | _, dnsWriter := p.Server.mailDeploymentFor() |
| 503 | for _, record := range domainRecords { |
| 504 | for _, member := range record.Apps { |
| 505 | if member == app { |
| 506 | estate = true |
| 507 | if record.Kind == mail.KindPlatform && len(record.Apps) == 1 && dnsWriter == nil { |
| 508 | return hostops.ErrContract |
| 509 | } |
| 510 | } |
| 511 | } |
| 512 | } |
| 513 | claims, err := mail.StrictDomainClaims(ctx, p.Controller.Store) |
| 514 | if err != nil { |
| 515 | return err |
| 516 | } |
| 517 | for _, claim := range claims { |
| 518 | if claim.Account == account && claim.App == app { |
| 519 | estate = true |
| 520 | } |
| 521 | } |
| 522 | if estate && p.Server.cfg.MailProvisioner == nil { |
| 523 | return hostops.ErrContract |
| 524 | } |
| 525 | return nil |
| 526 | } |
| 527 | |
| 528 | func (p *ClaimPurgeCoordinator) captureDomains(ctx context.Context, account, app, id, appID string) error { |
| 529 | w, err := p.control() |
| 530 | if err != nil { |
| 531 | return err |
| 532 | } |
| 533 | keys, err := p.Controller.Store.List(ctx, domains.ClaimsPrefix) |
| 534 | if err != nil { |
| 535 | return err |
| 536 | } |
| 537 | j, tag, err := hostops.ReadPurgeJournal(ctx, p.Controller.Store, account, app, id, appID) |
| 538 | if err != nil { |
| 539 | return err |
| 540 | } |
| 541 | if j.Phase != "reserved" { |
| 542 | return hostops.ErrPending |
| 543 | } |
| 544 | if j.Effects == nil { |
| 545 | j.Effects = map[string]json.RawMessage{} |
| 546 | } |
| 547 | for _, key := range keys { |
| 548 | raw, etag, err := p.Controller.Store.Get(ctx, key) |
| 549 | if err != nil { |
| 550 | return err |
| 551 | } |
| 552 | var claim domains.Claim |
| 553 | if err = json.Unmarshal(raw, &claim); err != nil { |
| 554 | return err |
| 555 | } |
| 556 | if claim.Account != account || claim.App != app { |
| 557 | continue |
| 558 | } |
| 559 | if claim.Acquisition == "" || claim.OwnershipState == "quarantined" || etag == "" || key != domains.ClaimKey(claim.Host) { |
| 560 | return hostops.ErrPending |
| 561 | } |
| 562 | captured, err := json.Marshal(hostops.PurgeObject{Key: key, ETag: etag, Body: raw}) |
| 563 | if err != nil { |
| 564 | return err |
| 565 | } |
| 566 | j.Effects["domain."+claim.Host] = captured |
| 567 | } |
| 568 | _, err = w.SavePurgeJournal(ctx, j, tag) |
| 569 | return err |
| 570 | } |
| 571 | |
| 572 | // retireDomains runs inside the "retire-domains" stage: its exact-ETag |
| 573 | // deletes pass the data boundary on that stage's permit. |
| 574 | func (p *ClaimPurgeCoordinator) retireDomains(ctx context.Context, account, app, id, appID string) error { |
| 575 | j, _, err := hostops.ReadPurgeJournal(ctx, p.Controller.Store, account, app, id, appID) |
| 576 | if err != nil { |
| 577 | return err |
| 578 | } |
| 579 | if j.Phase != "torn-down" { |
| 580 | return hostops.ErrPending |
| 581 | } |
| 582 | for _, host := range j.Hosts { |
| 583 | if host.Holder != nil && !host.Retired { |
| 584 | return hostops.ErrPending |
| 585 | } |
| 586 | } |
| 587 | for name, raw := range j.Effects { |
| 588 | if len(name) < 7 || name[:7] != "domain." { |
| 589 | continue |
| 590 | } |
| 591 | var captured hostops.PurgeObject |
| 592 | if err = json.Unmarshal(raw, &captured); err != nil { |
| 593 | return err |
| 594 | } |
| 595 | var old domains.Claim |
| 596 | if err = json.Unmarshal(captured.Body, &old); err != nil { |
| 597 | return err |
| 598 | } |
| 599 | if old.Account != account || old.App != app || old.Acquisition == "" || captured.Key != domains.ClaimKey(old.Host) { |
| 600 | return hostops.ErrIdentity |
| 601 | } |
| 602 | body, etag, err := p.Controller.Store.Get(ctx, captured.Key) |
| 603 | if errors.Is(err, os.ErrNotExist) { |
| 604 | continue |
| 605 | } |
| 606 | if err != nil { |
| 607 | return err |
| 608 | } |
| 609 | var current domains.Claim |
| 610 | if err = json.Unmarshal(body, ¤t); err != nil { |
| 611 | return err |
| 612 | } |
| 613 | if current.Acquisition != old.Acquisition { |
| 614 | continue |
| 615 | } |
| 616 | if etag != captured.ETag || !bytes.Equal(body, captured.Body) { |
| 617 | return hostops.ErrPending |
| 618 | } |
| 619 | if err = release.DeleteIfMatch(ctx, p.Controller.Store, captured.Key, captured.ETag); err != nil { |
| 620 | return err |
| 621 | } |
| 622 | } |
| 623 | return nil |
| 624 | } |
| 625 | |
| 626 | // teardown runs inside the "teardown" stage and the held mail gate. Each |
| 627 | // backend call is its own effect under the purge operation. |
| 628 | func (p *ClaimPurgeCoordinator) teardown(ctx context.Context, h *hostops.OperationHandle, account, app, id, appID string) error { |
| 629 | s := p.Server |
| 630 | w, err := p.control() |
| 631 | if err != nil { |
| 632 | return err |
| 633 | } |
| 634 | if err := p.mailReady(ctx, account, app); err != nil { |
| 635 | return err |
| 636 | } |
| 637 | j, tag, err := hostops.ReadPurgeJournal(ctx, p.Controller.Store, account, app, id, appID) |
| 638 | if err != nil { |
| 639 | return err |
| 640 | } |
| 641 | if j.Phase != "claimed" { |
| 642 | return hostops.ErrPending |
| 643 | } |
| 644 | if j.Effects == nil { |
| 645 | j.Effects = map[string]json.RawMessage{} |
| 646 | } |
| 647 | // Capture every backend reconstruction input under both gates, before |
| 648 | // the first destructive call. |
| 649 | envs, err := objstore.StrictStoreEnvironments(ctx, p.Controller.Store, account, app) |
| 650 | if err != nil { |
| 651 | return err |
| 652 | } |
| 653 | for _, env := range append(envs, "") { |
| 654 | record, _, err := objstore.StrictGetIn(ctx, p.Controller.Store, account, app, env) |
| 655 | if err != nil { |
| 656 | return err |
| 657 | } |
| 658 | if record != nil && (record.Provisioned != nil || record.Grant != nil) && s.cfg.StoreProvisioner == nil { |
| 659 | return hostops.ErrContract |
| 660 | } |
| 661 | raw, err := json.Marshal(record) |
| 662 | if err != nil { |
| 663 | return err |
| 664 | } |
| 665 | j.Effects["store."+env] = raw |
| 666 | } |
| 667 | record, _, err := mail.StrictGet(ctx, p.Controller.Store, account, app) |
| 668 | if err != nil { |
| 669 | return err |
| 670 | } |
| 671 | raw, err := json.Marshal(record) |
| 672 | if err != nil { |
| 673 | return err |
| 674 | } |
| 675 | j.Effects["email.record"] = raw |
| 676 | domainRecords, err := mail.StrictListDomains(ctx, p.Controller.Store, account) |
| 677 | if err != nil { |
| 678 | return err |
| 679 | } |
| 680 | for _, record := range domainRecords { |
| 681 | raw, err := json.Marshal(record) |
| 682 | if err != nil { |
| 683 | return err |
| 684 | } |
| 685 | j.Effects["email.domain."+record.Domain] = raw |
| 686 | } |
| 687 | domainClaims, err := mail.StrictDomainClaims(ctx, p.Controller.Store) |
| 688 | if err != nil { |
| 689 | return err |
| 690 | } |
| 691 | for _, claim := range domainClaims { |
| 692 | if claim.Account == account && claim.App == app { |
| 693 | raw, err := json.Marshal(claim) |
| 694 | if err != nil { |
| 695 | return err |
| 696 | } |
| 697 | j.Effects["email.claim."+claim.Domain] = raw |
| 698 | } |
| 699 | } |
| 700 | if _, err = w.SavePurgeJournal(ctx, j, tag); err != nil { |
| 701 | return err |
| 702 | } |
| 703 | effects := claimPurgeEffects{h: h} |
| 704 | for _, env := range append(envs, "") { |
| 705 | rec, _, err := objstore.StrictGetIn(ctx, p.Controller.Store, account, app, env) |
| 706 | if err != nil { |
| 707 | return err |
| 708 | } |
| 709 | if rec == nil || (rec.Provisioned == nil && rec.Grant == nil) { |
| 710 | continue |
| 711 | } |
| 712 | var scanner objstore.Scanner |
| 713 | if s.cfg.StoreScanner != nil { |
| 714 | scanner = claimPurgeScanner{raw: s.cfg.StoreScanner, e: effects} |
| 715 | } |
| 716 | if err = objstore.TeardownIn(ctx, p.Controller.Store, claimPurgeStore{raw: s.cfg.StoreProvisioner, e: effects}, scanner, account, app, env); err != nil { |
| 717 | return err |
| 718 | } |
| 719 | } |
| 720 | if s.cfg.MailProvisioner != nil { |
| 721 | deployment, writer := s.mailDeploymentFor() |
| 722 | var guardedDNS dnsrec.Writer |
| 723 | if writer != nil { |
| 724 | guardedDNS = claimPurgeDNS{raw: writer, e: effects} |
| 725 | } |
| 726 | if err = mail.Teardown(ctx, s.claimMailStore(), claimPurgeMail{raw: s.cfg.MailProvisioner, e: effects}, guardedDNS, deployment, account, app); err != nil { |
| 727 | return err |
| 728 | } |
| 729 | } |
| 730 | return nil |
| 731 | } |
| 732 | |
| 733 | func (s *Server) SweepClaimPurgeRequests(ctx context.Context) error { |
| 734 | if s.cfg.ClaimPurge == nil { |
| 735 | return hostops.ErrContract |
| 736 | } |
| 737 | return s.cfg.ClaimPurge.SweepRequests(ctx) |
| 738 | } |
| 739 | |