| 1 | package console |
| 2 | |
| 3 | // claim_restore.go is the flag-on half of restoreApp (restore.go). Both doors |
| 4 | // still call restoreApp, which hands over here when HostClaim is on; its |
| 5 | // errors are the same errNotInTrash, *temporaryCapError and *restoreFailure |
| 6 | // values, so each door's existing responses (and copy) are unchanged. |
| 7 | // |
| 8 | // The order, under the same account-then-app locks: |
| 9 | // |
| 10 | // 0. finish any interrupted restore of this app (ControlWriter.RecoverRestore) |
| 11 | // 1. project canonical state into this row (the flag-on settle) |
| 12 | // 2. not in the trash / already restored / cap check, as restoreApp |
| 13 | // 3. read the exact marker version |
| 14 | // 4. BeginExclusive (app-mutation, action restore) |
| 15 | // 5. PrepareRestore: the receipt, with the marker ETag, BEFORE any effect is |
| 16 | // issued and before any SQL |
| 17 | // 6. Issue the operation's one effect (the live permit) |
| 18 | // 7. under the effect permit: restoring = 1 (this console's slot) |
| 19 | // 8. DeleteIfMatch of exactly that version, through the data boundary |
| 20 | // 9. CompleteRestore, settle the effect, FinishMutation |
| 21 | // 10. under the receipt grant: this console's row (FinishClaimRestoreRow) |
| 22 | // |
| 23 | // Steps 5, 8 and 9 are the global effect. Once step 5 has saved the receipt |
| 24 | // the restore is committed: any console's RecoverMutation finishes the |
| 25 | // receipt's exact-ETag delete (which cannot remove a successor marker), and |
| 26 | // AbandonRestore refuses it. Before step 5 nothing was issued, so a failure |
| 27 | // just reopens. The expiry bump is projection, folded into step 10, which |
| 28 | // every console (this one included) replays from the completed receipt in |
| 29 | // materializeClaimApp. Nothing here writes SQL except through |
| 30 | // Store.WithAppMutation, and nothing writes canonical admission except the |
| 31 | // typed ControlWriter. |
| 32 | |
| 33 | import ( |
| 34 | "context" |
| 35 | "crypto/rand" |
| 36 | "database/sql" |
| 37 | "encoding/hex" |
| 38 | "errors" |
| 39 | "fmt" |
| 40 | "log" |
| 41 | "strconv" |
| 42 | "time" |
| 43 | |
| 44 | "github.com/carlosframework/platform/internal/hostops" |
| 45 | "github.com/carlosframework/platform/internal/release" |
| 46 | ) |
| 47 | |
| 48 | // claimRestoreAbandonAfter is how long a restore operation that never saved |
| 49 | // its receipt is left to its issuer before another console abandons it (see |
| 50 | // hostops/restore_recover.go for why abandoning is a fence). |
| 51 | const claimRestoreAbandonAfter = 2 * time.Minute |
| 52 | |
| 53 | // errClaimRestoreUnconfirmed is a trashed row with no marker and no completed |
| 54 | // receipt: canonical state does not say this app is in the trash, so there |
| 55 | // is nothing a restore may delete. |
| 56 | var errClaimRestoreUnconfirmed = errors.New("console: the trash marker is missing and no restore receipt confirms one") |
| 57 | |
| 58 | func newClaimRestoreID() (string, error) { |
| 59 | var b [16]byte |
| 60 | if _, err := rand.Read(b[:]); err != nil { |
| 61 | return "", err |
| 62 | } |
| 63 | return "restore-" + hex.EncodeToString(b[:]), nil |
| 64 | } |
| 65 | |
| 66 | // claimRestoreRuntime is the typed control writer and this console's |
| 67 | // controller identity, as ConfigureClaimRuntime installed them. Anything |
| 68 | // missing, or an ungated database, refuses. |
| 69 | func (s *Server) claimRestoreRuntime() (*hostops.ControlWriter, hostops.RegistrationOwner, error) { |
| 70 | a := s.cfg.ClaimAuthority |
| 71 | if a == nil || a.Control == nil || a.ConsoleID == "" || a.Enrollment == "" || s.cfg.Releases == nil || !s.store.claimGated() { |
| 72 | return nil, hostops.RegistrationOwner{}, hostops.ErrContract |
| 73 | } |
| 74 | return a.Control, hostops.RegistrationOwner{Console: a.ConsoleID, Enrollment: a.Enrollment}, nil |
| 75 | } |
| 76 | |
| 77 | // restoreClaimApp is restoreApp with HostClaim on. The caller has checked |
| 78 | // s.cfg.Releases != nil. |
| 79 | func (s *Server) restoreClaimApp(ctx context.Context, acct *Account, app string, now time.Time) error { |
| 80 | w, self, err := s.claimRestoreRuntime() |
| 81 | if err != nil { |
| 82 | return &restoreFailure{Marker: true, Err: err} |
| 83 | } |
| 84 | defer s.lockAccount(acct.Sqid)() |
| 85 | defer s.lockApp(acct.Sqid, app)() |
| 86 | appID, err := s.store.ClaimAppID(acct.ID, app) |
| 87 | if errors.Is(err, sql.ErrNoRows) { |
| 88 | return errNotInTrash |
| 89 | } |
| 90 | if err != nil { |
| 91 | return &restoreFailure{Marker: true, Err: err} |
| 92 | } |
| 93 | prior, err := s.store.AppLife(acct.ID, app) |
| 94 | if err != nil { |
| 95 | return &restoreFailure{Marker: true, Err: err} |
| 96 | } |
| 97 | // 0. ErrPending is a young operation another console still owns; step 4 |
| 98 | // refuses it below. |
| 99 | if _, err := w.RecoverRestore(ctx, acct.Sqid, app, appID, now, claimRestoreAbandonAfter); err != nil && !errors.Is(err, hostops.ErrPending) { |
| 100 | return &restoreFailure{Marker: true, Err: err} |
| 101 | } |
| 102 | // 1. ErrPending is a projection that must wait (an operation owns the |
| 103 | // app, or a receipt is not this row's); the checks below decide. |
| 104 | if _, err := materializeClaimApp(ctx, s.cfg.Releases, s.store, acct.ID, acct.Sqid, app, now); err != nil && !errors.Is(err, hostops.ErrPending) { |
| 105 | return &restoreFailure{Marker: true, Err: err} |
| 106 | } |
| 107 | life, err := s.store.AppLife(acct.ID, app) |
| 108 | if err != nil { |
| 109 | return &restoreFailure{Marker: true, Err: err} |
| 110 | } |
| 111 | // 2. As restoreApp: a restore finished by step 0 or 1 is this restore, |
| 112 | // answered as one. |
| 113 | if prior != nil && prior.Trashed() && life != nil && !life.Trashed() { |
| 114 | return nil |
| 115 | } |
| 116 | if life == nil || !life.Trashed() { |
| 117 | return errNotInTrash |
| 118 | } |
| 119 | // Plans 3a: restoring brings a running thing back, so it meets the cap. |
| 120 | if err := s.checkRestore(ctx, acct, app); err != nil { |
| 121 | if isPlanCap(err) { |
| 122 | return err |
| 123 | } |
| 124 | return &restoreFailure{Marker: true, Err: err} |
| 125 | } |
| 126 | if life.Temporary() { |
| 127 | if err := s.temporaryCapCheck(acct.ID); err != nil { |
| 128 | var capErr *temporaryCapError |
| 129 | if errors.As(err, &capErr) { |
| 130 | return capErr |
| 131 | } |
| 132 | return &restoreFailure{Marker: true, Err: err} |
| 133 | } |
| 134 | } |
| 135 | // 3. |
| 136 | tomb, etag, err := readTombstoneVersion(ctx, s.cfg.Releases, acct.Sqid, app) |
| 137 | if err != nil { |
| 138 | return &restoreFailure{Marker: true, Err: err} |
| 139 | } |
| 140 | if tomb == nil { |
| 141 | return &restoreFailure{Marker: true, Err: errClaimRestoreUnconfirmed} |
| 142 | } |
| 143 | // 4. |
| 144 | op, err := newClaimRestoreID() |
| 145 | if err != nil { |
| 146 | return &restoreFailure{Marker: true, Err: err} |
| 147 | } |
| 148 | owner := self |
| 149 | owner.Controller = op |
| 150 | h, err := w.BeginExclusive(ctx, acct.Sqid, app, appID, hostops.AppOperation{ID: op, Kind: hostops.OpAppMutation, AppID: appID, Phase: "issued", |
| 151 | Intent: map[string]string{"action": "restore", hostops.RestoreActionStarted: strconv.FormatInt(now.Unix(), 10)}}, owner) |
| 152 | if err != nil { |
| 153 | return &restoreFailure{Marker: true, Err: err} |
| 154 | } |
| 155 | // 5. The receipt, before any effect or SQL. A lost response may still |
| 156 | // have saved it; only an operation with no receipt (and so nothing |
| 157 | // issued) is reopened here. |
| 158 | if err := h.PrepareRestore(ctx, etag, tomb.DeletedAt.Unix()); err != nil { |
| 159 | current, rerr := hostops.ReadAdmission(ctx, s.cfg.Releases, acct.Sqid, app, appID) |
| 160 | if rerr != nil || current.Value.LastRestore == nil || current.Value.LastRestore.ID != op { |
| 161 | return &restoreFailure{Marker: true, Err: errors.Join(err, rerr, h.FinishMutation(context.WithoutCancel(ctx)))} |
| 162 | } |
| 163 | } |
| 164 | // 6. From here the receipt commits the restore: a failure is finished by |
| 165 | // the receipt's recovery, never abandoned. |
| 166 | pctx, settle, err := h.Runner(time.Now).Issue(ctx, hostops.Effect{ID: op, Resource: hostops.OpAppMutation, Action: "restore", |
| 167 | Inputs: map[string]string{"tombstone_etag": etag}}) |
| 168 | if err != nil { |
| 169 | if ferr := s.recoverClaimRestoreReceipt(ctx, w, acct, app, appID, op, err); ferr != nil { |
| 170 | return ferr |
| 171 | } |
| 172 | } else { |
| 173 | // 7. This console's restoring slot, under the live permit. A failure |
| 174 | // here is local projection only; the delete still runs, and step 10 |
| 175 | // (or any later materialization) clears the flag. |
| 176 | scope := claimSQLScope{AccountID: acct.ID, App: app, AppID: appID} |
| 177 | if err := s.store.WithAppMutation(ctx, []sqlGrant{permitGrant(pctx, s.cfg.Releases, acct.Sqid, scope)}, func(v *Store) error { |
| 178 | _, err := v.SetClaimRestoring(acct.ID, app, appID, true) |
| 179 | return err |
| 180 | }); err != nil { |
| 181 | log.Printf("console: claim restore %s/%s: restoring slot: %v", acct.Sqid, app, err) |
| 182 | } |
| 183 | // 8 and 9. |
| 184 | derr := release.DeleteIfMatch(pctx, s.cfg.Releases, release.AppTombstoneKey(acct.Sqid, app), etag) |
| 185 | var cerr error |
| 186 | if derr == nil { |
| 187 | cerr = h.CompleteRestore(ctx) |
| 188 | } |
| 189 | if stepErr := errors.Join(derr, cerr); stepErr != nil { |
| 190 | // Revoke the permit and record uncertainty, then let the bucket |
| 191 | // decide: RecoverMutation finishes the receipt's exact delete. |
| 192 | settleErr := settle("", stepErr) |
| 193 | if ferr := s.recoverClaimRestoreReceipt(ctx, w, acct, app, appID, op, errors.Join(stepErr, settleErr)); ferr != nil { |
| 194 | return ferr |
| 195 | } |
| 196 | } else { |
| 197 | if err := settle("restored", nil); err != nil { |
| 198 | // The marker is gone and the receipt is complete; the sweep's |
| 199 | // RecoverMutation settles the effect and reopens. |
| 200 | return &restoreFailure{Marker: false, Err: err} |
| 201 | } |
| 202 | if err := h.FinishMutation(ctx); err != nil { |
| 203 | return &restoreFailure{Marker: false, Err: err} |
| 204 | } |
| 205 | } |
| 206 | } |
| 207 | // 10. This console's projection, from the completed receipt. |
| 208 | current, err := hostops.ReadAdmission(ctx, s.cfg.Releases, acct.Sqid, app, appID) |
| 209 | if err != nil { |
| 210 | return &restoreFailure{Marker: false, Err: err} |
| 211 | } |
| 212 | receipt := current.Value.LastRestore |
| 213 | if receipt == nil || receipt.ID != op || !restoreReceiptCurrent(current.Value, *receipt) { |
| 214 | return &restoreFailure{Marker: false, Err: fmt.Errorf("console: restore %s/%s: receipt %s is not settled: %w", acct.Sqid, app, op, hostops.ErrPending)} |
| 215 | } |
| 216 | _, err = materializeClaimApp(ctx, s.cfg.Releases, s.store, acct.ID, acct.Sqid, app, now) |
| 217 | if err != nil { |
| 218 | return &restoreFailure{Marker: false, Err: err} |
| 219 | } |
| 220 | return nil |
| 221 | } |
| 222 | |
| 223 | // recoverClaimRestoreReceipt finishes a restore whose receipt is saved but |
| 224 | // whose issuer path failed (stepErr): RecoverMutation replays the receipt's |
| 225 | // exact-ETag delete through the control writer and reopens. nil means the |
| 226 | // restore finished and step 10 may run; otherwise the door's failure, with |
| 227 | // Marker set while any marker still hides the app. |
| 228 | func (s *Server) recoverClaimRestoreReceipt(ctx context.Context, w *hostops.ControlWriter, acct *Account, app, appID, op string, stepErr error) error { |
| 229 | rerr := w.RecoverMutation(context.WithoutCancel(ctx), acct.Sqid, app, appID, op) |
| 230 | if rerr == nil { |
| 231 | return nil |
| 232 | } |
| 233 | log.Printf("console: claim restore %s/%s: operation %s keeps its receipt for recovery: %v", acct.Sqid, app, op, errors.Join(stepErr, rerr)) |
| 234 | _, seen, terr := readTombstoneVersion(ctx, s.cfg.Releases, acct.Sqid, app) |
| 235 | if terr != nil || seen != "" { // any marker still hides the app everywhere |
| 236 | return &restoreFailure{Marker: true, Err: errors.Join(stepErr, rerr, terr)} |
| 237 | } |
| 238 | return &restoreFailure{Marker: false, Err: errors.Join(stepErr, rerr)} |
| 239 | } |
| 240 | |
| 241 | // reconcileClaimRestore is the admission sweep's restore case: finish a |
| 242 | // saved receipt, or abandon an aged receipt-less restore. It never projects |
| 243 | // SQL; each console's materialization replays the completed receipt. |
| 244 | func (s *Server) reconcileClaimRestore(ctx context.Context, account, app, appID string) error { |
| 245 | w, _, err := s.claimRestoreRuntime() |
| 246 | if err != nil { |
| 247 | return err |
| 248 | } |
| 249 | out, err := w.RecoverRestore(ctx, account, app, appID, time.Now(), claimRestoreAbandonAfter) |
| 250 | if err != nil { |
| 251 | return err |
| 252 | } |
| 253 | if out != hostops.RestoreNone { |
| 254 | log.Printf("console: claim restore %s/%s: recovered (%s)", account, app, out) |
| 255 | } |
| 256 | return nil |
| 257 | } |
| 258 | |
amadan